feat(store,api): archive with full-text search, threading and archive policy
- Add archive table with FTS5 index kept in sync by triggers - Deduplicate by Message-ID per list so MTA retries do not repeat posts - Thread messages through In-Reply-To within the same list - Quote every search word so user input is never FTS syntax - Add archive_policy (public, private, off) with migration - Add API to list, search, read and delete archived messages
This commit is contained in:
parent
aad87be74c
commit
d8275aaa1f
7 changed files with 672 additions and 33 deletions
|
|
@ -45,6 +45,9 @@ func New(s *store.Store, token string, rel Releaser) http.Handler {
|
|||
mux.Handle("GET /api/v1/lists/{id}/members", srv.auth(srv.listMembers))
|
||||
mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember))
|
||||
mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember))
|
||||
mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive))
|
||||
mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived))
|
||||
mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived))
|
||||
mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld))
|
||||
mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld))
|
||||
mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld))
|
||||
|
|
@ -72,13 +75,14 @@ type listDTO struct {
|
|||
Title string `json:"title"`
|
||||
Address string `json:"address"`
|
||||
|
||||
PostPolicy string `json:"post_policy"`
|
||||
JoinPolicy string `json:"join_policy"`
|
||||
PostPolicy string `json:"post_policy"`
|
||||
JoinPolicy string `json:"join_policy"`
|
||||
ArchivePolicy string `json:"archive_policy"`
|
||||
}
|
||||
|
||||
func toListDTO(l store.List) listDTO {
|
||||
return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address(),
|
||||
PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy}
|
||||
PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy, ArchivePolicy: l.ArchivePolicy}
|
||||
}
|
||||
|
||||
type memberDTO struct {
|
||||
|
|
@ -195,11 +199,13 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) {
|
|||
Title *string `json:"title"`
|
||||
PostPolicy *string `json:"post_policy"`
|
||||
JoinPolicy *string `json:"join_policy"`
|
||||
|
||||
ArchivePolicy *string `json:"archive_policy"`
|
||||
}
|
||||
if !decodeBody(w, r, &in) {
|
||||
return
|
||||
}
|
||||
if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil {
|
||||
if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil && in.ArchivePolicy == nil {
|
||||
writeError(w, http.StatusBadRequest, "nothing to update")
|
||||
return
|
||||
}
|
||||
|
|
@ -222,17 +228,21 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) {
|
|||
if in.JoinPolicy != nil {
|
||||
l.JoinPolicy = *in.JoinPolicy
|
||||
}
|
||||
if in.ArchivePolicy != nil {
|
||||
l.ArchivePolicy = *in.ArchivePolicy
|
||||
}
|
||||
// Validate before writing anything.
|
||||
if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated) ||
|
||||
(l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) {
|
||||
writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy must be open or closed")
|
||||
(l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) ||
|
||||
(l.ArchivePolicy != store.ArchivePublic && l.ArchivePolicy != store.ArchivePrivate && l.ArchivePolicy != store.ArchiveOff) {
|
||||
writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy open or closed; archive_policy public, private or off")
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateList(id, l.Title); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy); err != nil {
|
||||
if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy, l.ArchivePolicy); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
|
|
@ -385,3 +395,99 @@ func (s *server) discardHeld(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// Handlers: archive.
|
||||
|
||||
const (
|
||||
defaultPageSize = 50
|
||||
maxPageSize = 200
|
||||
)
|
||||
|
||||
type messageDTO struct {
|
||||
ID int64 `json:"id"`
|
||||
ThreadID int64 `json:"thread_id"`
|
||||
MessageID string `json:"message_id"`
|
||||
InReplyTo string `json:"in_reply_to,omitempty"`
|
||||
Subject string `json:"subject"`
|
||||
Sender string `json:"sender"`
|
||||
Date time.Time `json:"date"`
|
||||
Size int `json:"size"`
|
||||
Body string `json:"body,omitempty"`
|
||||
}
|
||||
|
||||
func toMessageDTO(m store.Message) messageDTO {
|
||||
return messageDTO{ID: m.ID, ThreadID: m.ThreadID, MessageID: m.MessageID, InReplyTo: m.InReplyTo,
|
||||
Subject: m.Subject, Sender: m.Sender, Date: m.Date, Size: m.Size, Body: m.Body}
|
||||
}
|
||||
|
||||
// intParam reads a non-negative integer query parameter.
|
||||
func intParam(r *http.Request, name string, def int) (int, bool) {
|
||||
v := r.URL.Query().Get(name)
|
||||
if v == "" {
|
||||
return def, true
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
return n, err == nil && n >= 0
|
||||
}
|
||||
|
||||
func (s *server) listArchive(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := listID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, ok1 := intParam(r, "limit", defaultPageSize)
|
||||
offset, ok2 := intParam(r, "offset", 0)
|
||||
if !ok1 || !ok2 {
|
||||
writeError(w, http.StatusBadRequest, "limit and offset must be non-negative integers")
|
||||
return
|
||||
}
|
||||
limit = min(max(limit, 1), maxPageSize)
|
||||
if _, err := s.store.GetList(id); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
var (
|
||||
ms []store.Message
|
||||
total int
|
||||
err error
|
||||
)
|
||||
if q := strings.TrimSpace(r.URL.Query().Get("q")); q != "" {
|
||||
ms, total, err = s.store.SearchMessages(id, q, limit, offset)
|
||||
} else {
|
||||
ms, total, err = s.store.ListMessages(id, limit, offset)
|
||||
}
|
||||
if err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
out := make([]messageDTO, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
out = append(out, toMessageDTO(m))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"total": total, "messages": out})
|
||||
}
|
||||
|
||||
func (s *server) getArchived(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := listID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
m, err := s.store.GetMessage(id)
|
||||
if err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, toMessageDTO(m))
|
||||
}
|
||||
|
||||
func (s *server) deleteArchived(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := listID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := s.store.DeleteMessage(id); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@ package api
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
|
@ -78,13 +80,14 @@ func TestEmptyTokenRejectsEverything(t *testing.T) {
|
|||
}
|
||||
|
||||
type listJSON struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Domain string `json:"domain"`
|
||||
Title string `json:"title"`
|
||||
Address string `json:"address"`
|
||||
PostPolicy string `json:"post_policy"`
|
||||
JoinPolicy string `json:"join_policy"`
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Domain string `json:"domain"`
|
||||
Title string `json:"title"`
|
||||
Address string `json:"address"`
|
||||
PostPolicy string `json:"post_policy"`
|
||||
JoinPolicy string `json:"join_policy"`
|
||||
ArchivePolicy string `json:"archive_policy"`
|
||||
}
|
||||
|
||||
func createList(t *testing.T, h http.Handler) listJSON {
|
||||
|
|
@ -315,3 +318,103 @@ func TestApproveUnknownHeld(t *testing.T) {
|
|||
type errReleaser struct{ err error }
|
||||
|
||||
func (e errReleaser) Release(context.Context, int64) error { return e.err }
|
||||
|
||||
func archiveServer(t *testing.T) http.Handler {
|
||||
t.Helper()
|
||||
s, _ := store.Open(":memory:")
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
for i, subj := range []string{"First post", "Second post", "Lunch"} {
|
||||
s.ArchiveMessage(l.ID, store.Message{
|
||||
MessageID: fmt.Sprintf("<%d@x>", i), Subject: subj, Sender: "Alice",
|
||||
Date: time.Unix(1_700_000_000+int64(i), 0), Body: "body of " + subj,
|
||||
})
|
||||
}
|
||||
return New(s, token, nil)
|
||||
}
|
||||
|
||||
type archiveJSON struct {
|
||||
Total int `json:"total"`
|
||||
Messages []struct {
|
||||
ID int64 `json:"id"`
|
||||
ThreadID int64 `json:"thread_id"`
|
||||
Subject string `json:"subject"`
|
||||
Sender string `json:"sender"`
|
||||
Body string `json:"body"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
|
||||
func TestArchivePolicyViaAPI(t *testing.T) {
|
||||
h := newServer(t)
|
||||
l := createList(t, h)
|
||||
if l.ArchivePolicy != "private" {
|
||||
t.Fatalf("default = %q", l.ArchivePolicy)
|
||||
}
|
||||
rec := do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"public"}`, true)
|
||||
expect(t, rec, 200)
|
||||
var u listJSON
|
||||
decode(t, rec, &u)
|
||||
if u.ArchivePolicy != "public" || u.PostPolicy != "members" {
|
||||
t.Fatalf("patched: %+v", u)
|
||||
}
|
||||
expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"bogus"}`, true), 400)
|
||||
}
|
||||
|
||||
func TestArchiveList(t *testing.T) {
|
||||
h := archiveServer(t)
|
||||
rec := do(t, h, "GET", "/api/v1/lists/1/archive", "", true)
|
||||
expect(t, rec, 200)
|
||||
var out archiveJSON
|
||||
decode(t, rec, &out)
|
||||
if out.Total != 3 || len(out.Messages) != 3 || out.Messages[0].Subject != "Lunch" {
|
||||
t.Fatalf("archive: %+v", out)
|
||||
}
|
||||
if out.Messages[0].Body != "" {
|
||||
t.Fatal("listings must not include bodies")
|
||||
}
|
||||
|
||||
rec = do(t, h, "GET", "/api/v1/lists/1/archive?limit=2&offset=2", "", true)
|
||||
decode(t, rec, &out)
|
||||
if out.Total != 3 || len(out.Messages) != 1 || out.Messages[0].Subject != "First post" {
|
||||
t.Fatalf("page: %+v", out)
|
||||
}
|
||||
|
||||
rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=post", "", true)
|
||||
decode(t, rec, &out)
|
||||
if out.Total != 2 || len(out.Messages) != 2 {
|
||||
t.Fatalf("search: %+v", out)
|
||||
}
|
||||
rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=nothingmatches", "", true)
|
||||
decode(t, rec, &out)
|
||||
if out.Total != 0 || out.Messages == nil || len(out.Messages) != 0 {
|
||||
t.Fatalf("empty search must be an empty array: %+v", out)
|
||||
}
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=abc", "", true), 400)
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?offset=-1", "", true), 400)
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/99/archive", "", true), 404)
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive", "", false), 401)
|
||||
}
|
||||
|
||||
func TestArchiveLimitIsCapped(t *testing.T) {
|
||||
h := archiveServer(t)
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=100000", "", true), 200)
|
||||
}
|
||||
|
||||
func TestArchiveGetAndDelete(t *testing.T) {
|
||||
h := archiveServer(t)
|
||||
rec := do(t, h, "GET", "/api/v1/archive/1", "", true)
|
||||
expect(t, rec, 200)
|
||||
var m struct {
|
||||
Subject string `json:"subject"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
decode(t, rec, &m)
|
||||
if m.Subject != "First post" || m.Body != "body of First post" {
|
||||
t.Fatalf("message: %+v", m)
|
||||
}
|
||||
expect(t, do(t, h, "GET", "/api/v1/archive/99", "", true), 404)
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 204)
|
||||
expect(t, do(t, h, "GET", "/api/v1/archive/1", "", true), 404)
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 404)
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/archive/2", "", false), 401)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -180,7 +180,7 @@ func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) {
|
|||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen)
|
||||
s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
||||
relay := &outbound.Fake{}
|
||||
srv := NewServer(router.New(s), relay)
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ func TestDisabledMembersAreSkipped(t *testing.T) {
|
|||
|
||||
func TestModeratedListHoldsPosts(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen)
|
||||
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
||||
for _, sender := range []string{"alice@example.com", "stranger@x.com"} {
|
||||
_, err := e.r.Route(sender, "dev@example.org", []byte(sample))
|
||||
if !errors.Is(err, ErrHeld) {
|
||||
|
|
@ -103,7 +103,7 @@ func TestModeratedListHoldsPosts(t *testing.T) {
|
|||
|
||||
func TestReleaseHeld(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen)
|
||||
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
||||
e.r.Route("stranger@x.com", "dev@example.org", []byte(sample))
|
||||
held, _ := e.s.ListHeld(e.list.ID)
|
||||
|
||||
|
|
@ -182,7 +182,7 @@ func TestJoinRateLimitedAndSilentForMembers(t *testing.T) {
|
|||
|
||||
func TestJoinClosed(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed)
|
||||
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate)
|
||||
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
if !errors.Is(err, ErrJoinClosed) {
|
||||
t.Fatalf("want ErrJoinClosed, got %v", err)
|
||||
|
|
@ -196,7 +196,7 @@ func TestConfirmAfterListClosed(t *testing.T) {
|
|||
e := newEnv(t)
|
||||
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
tok := token(t, lastMail(t, e.out))
|
||||
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed)
|
||||
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate)
|
||||
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
|
||||
if !errors.Is(err, ErrJoinClosed) {
|
||||
t.Fatalf("want ErrJoinClosed, got %v", err)
|
||||
|
|
|
|||
229
internal/store/archive_test.go
Normal file
229
internal/store/archive_test.go
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func msg(id, subject, body string) Message {
|
||||
return Message{MessageID: id, Subject: subject, Sender: "Alice", Date: time.Unix(1_700_000_000, 0), Body: body}
|
||||
}
|
||||
|
||||
func TestArchivePolicyDefaultsAndUpdate(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if l.ArchivePolicy != ArchivePrivate {
|
||||
t.Fatalf("default = %q", l.ArchivePolicy)
|
||||
}
|
||||
if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, ArchivePublic); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := s.GetList(l.ID)
|
||||
if got.ArchivePolicy != ArchivePublic {
|
||||
t.Fatalf("policy = %q", got.ArchivePolicy)
|
||||
}
|
||||
if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, "bogus"); err == nil {
|
||||
t.Fatal("invalid archive policy must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveAndFetch(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
m := msg("<1@x>", "Hello", "first body")
|
||||
id, created, err := s.ArchiveMessage(l.ID, m)
|
||||
if err != nil || !created || id == 0 {
|
||||
t.Fatalf("archive: %d %v %v", id, created, err)
|
||||
}
|
||||
got, err := s.GetMessage(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Subject != "Hello" || got.Body != "first body" || got.ListID != l.ID || got.ThreadID != id || got.Size != len("first body") {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
if !got.Date.Equal(m.Date) {
|
||||
t.Fatalf("date = %v", got.Date)
|
||||
}
|
||||
if _, err := s.GetMessage(999); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveDeduplicatesByMessageID(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
id1, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a"))
|
||||
id2, created, err := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a"))
|
||||
if err != nil || created || id2 != id1 {
|
||||
t.Fatalf("retry must not duplicate: %d %v %v", id2, created, err)
|
||||
}
|
||||
other, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
if _, created, _ := s.ArchiveMessage(other.ID, msg("<1@x>", "Hello", "a")); !created {
|
||||
t.Fatal("same Message-ID in another list is a different message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreading(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
root, _, _ := s.ArchiveMessage(l.ID, msg("<root@x>", "Topic", "r"))
|
||||
reply := msg("<r1@x>", "Re: Topic", "r1")
|
||||
reply.InReplyTo = "<root@x>"
|
||||
r1, _, _ := s.ArchiveMessage(l.ID, reply)
|
||||
nested := msg("<r2@x>", "Re: Topic", "r2")
|
||||
nested.InReplyTo = "<r1@x>"
|
||||
r2, _, _ := s.ArchiveMessage(l.ID, nested)
|
||||
orphan := msg("<o@x>", "Other", "o")
|
||||
orphan.InReplyTo = "<missing@x>"
|
||||
o, _, _ := s.ArchiveMessage(l.ID, orphan)
|
||||
|
||||
for _, id := range []int64{r1, r2} {
|
||||
m, _ := s.GetMessage(id)
|
||||
if m.ThreadID != root {
|
||||
t.Fatalf("message %d thread = %d, want %d", id, m.ThreadID, root)
|
||||
}
|
||||
}
|
||||
if m, _ := s.GetMessage(o); m.ThreadID != o {
|
||||
t.Fatalf("orphan must start its own thread, got %d", m.ThreadID)
|
||||
}
|
||||
th, err := s.ThreadMessages(l.ID, root)
|
||||
if err != nil || len(th) != 3 {
|
||||
t.Fatalf("thread: %d %v", len(th), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreadingIgnoresOtherLists(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
a, _ := s.CreateList("a", "example.org", "A")
|
||||
b, _ := s.CreateList("b", "example.org", "B")
|
||||
s.ArchiveMessage(a.ID, msg("<root@x>", "Topic", "r"))
|
||||
reply := msg("<r@x>", "Re: Topic", "r")
|
||||
reply.InReplyTo = "<root@x>"
|
||||
id, _, _ := s.ArchiveMessage(b.ID, reply)
|
||||
if m, _ := s.GetMessage(id); m.ThreadID != id {
|
||||
t.Fatal("a parent in another list must not be used")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMessagesNewestFirstWithPaging(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
for i := 0; i < 5; i++ {
|
||||
m := msg("<m"+string(rune('a'+i))+"@x>", "S"+string(rune('a'+i)), "body")
|
||||
m.Date = time.Unix(1_700_000_000+int64(i), 0)
|
||||
s.ArchiveMessage(l.ID, m)
|
||||
}
|
||||
got, total, err := s.ListMessages(l.ID, 2, 0)
|
||||
if err != nil || total != 5 || len(got) != 2 {
|
||||
t.Fatalf("page 1: %d %d %v", len(got), total, err)
|
||||
}
|
||||
if got[0].Subject != "Se" || got[1].Subject != "Sd" {
|
||||
t.Fatalf("order: %s %s", got[0].Subject, got[1].Subject)
|
||||
}
|
||||
if got[0].Body != "" {
|
||||
t.Fatal("listings must not load bodies")
|
||||
}
|
||||
got, _, _ = s.ListMessages(l.ID, 2, 4)
|
||||
if len(got) != 1 || got[0].Subject != "Sa" {
|
||||
t.Fatalf("last page: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearch(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
other, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
s.ArchiveMessage(l.ID, msg("<1@x>", "Deploy failed", "the kubernetes rollout broke"))
|
||||
s.ArchiveMessage(l.ID, msg("<2@x>", "Lunch", "pizza on friday"))
|
||||
s.ArchiveMessage(other.ID, msg("<3@x>", "Deploy ok", "kubernetes is fine here"))
|
||||
|
||||
got, total, err := s.SearchMessages(l.ID, "kubernetes", 10, 0)
|
||||
if err != nil || total != 1 || len(got) != 1 || got[0].Subject != "Deploy failed" {
|
||||
t.Fatalf("search: %+v %d %v", got, total, err)
|
||||
}
|
||||
// Subject, sender and multiple terms (AND).
|
||||
if got, _, _ := s.SearchMessages(l.ID, "lunch pizza", 10, 0); len(got) != 1 {
|
||||
t.Fatalf("subject+body: %+v", got)
|
||||
}
|
||||
if got, _, _ := s.SearchMessages(l.ID, "alice", 10, 0); len(got) != 2 {
|
||||
t.Fatalf("sender: %+v", got)
|
||||
}
|
||||
if got, _, _ := s.SearchMessages(l.ID, "lunch kubernetes", 10, 0); len(got) != 0 {
|
||||
t.Fatalf("AND semantics: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchIsSafeAgainstSyntax(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "world"))
|
||||
for _, q := range []string{`"`, `foo AND`, `NEAR(`, `*`, `a OR b`, `col:x`, `)(`, "", " "} {
|
||||
if _, _, err := s.SearchMessages(l.ID, q, 10, 0); err != nil {
|
||||
t.Errorf("query %q must not error: %v", q, err)
|
||||
}
|
||||
}
|
||||
// FTS operators are plain words: "world OR zzz" needs both terms.
|
||||
if got, _, _ := s.SearchMessages(l.ID, "world OR zzz", 10, 0); len(got) != 0 {
|
||||
t.Fatalf("operators must be literal: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMessageRemovesFromSearch(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
id, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Secret", "leaked password"))
|
||||
if err := s.DeleteMessage(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _, _ := s.SearchMessages(l.ID, "password", 10, 0); len(got) != 0 {
|
||||
t.Fatalf("deleted message still searchable: %+v", got)
|
||||
}
|
||||
if err := s.DeleteMessage(id); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteListCleansArchiveAndIndex(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "uniqueword"))
|
||||
if err := s.DeleteList(l.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var n int
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts WHERE archive_fts MATCH 'uniqueword'`).Scan(&n)
|
||||
if n != 0 {
|
||||
t.Fatal("search index must follow cascade deletes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationAddsArchiveColumn(t *testing.T) {
|
||||
// A database from milestone 2 has no archive_policy column.
|
||||
path := t.TempDir() + "/m2.db"
|
||||
s, _ := Open(path)
|
||||
s.CreateList("dev", "example.org", "Dev")
|
||||
s.db.Exec(`ALTER TABLE lists DROP COLUMN archive_policy`)
|
||||
s.Close()
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
l, err := s2.FindList("dev@example.org")
|
||||
if err != nil || l.ArchivePolicy != ArchivePrivate {
|
||||
t.Fatalf("list: %+v %v", l, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongBodyIsStoredAsIs(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
body := strings.Repeat("x", 100_000)
|
||||
id, _, _ := s.ArchiveMessage(l.ID, msg("<big@x>", "Big", body))
|
||||
if got, _ := s.GetMessage(id); len(got.Body) != len(body) {
|
||||
t.Fatalf("body len %d", len(got.Body))
|
||||
}
|
||||
}
|
||||
|
|
@ -14,17 +14,17 @@ func TestListPolicies(t *testing.T) {
|
|||
if l.PostPolicy != "members" || l.JoinPolicy != "open" {
|
||||
t.Fatalf("defaults: %+v", l)
|
||||
}
|
||||
if err := s.SetPolicies(l.ID, "moderated", "closed"); err != nil {
|
||||
if err := s.SetPolicies(l.ID, "moderated", "closed", "private"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := s.GetList(l.ID)
|
||||
if got.PostPolicy != "moderated" || got.JoinPolicy != "closed" {
|
||||
t.Fatalf("policies: %+v", got)
|
||||
}
|
||||
if err := s.SetPolicies(l.ID, "bogus", "open"); err == nil {
|
||||
if err := s.SetPolicies(l.ID, "bogus", "open", "private"); err == nil {
|
||||
t.Fatal("invalid policy must be rejected")
|
||||
}
|
||||
if err := s.SetPolicies(999, "members", "open"); !errors.Is(err, ErrNotFound) {
|
||||
if err := s.SetPolicies(999, "members", "open", "private"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,13 @@ const (
|
|||
JoinClosed = "closed" // only the admin adds members
|
||||
)
|
||||
|
||||
// Archive policy values.
|
||||
const (
|
||||
ArchivePublic = "public" // browsable on the web
|
||||
ArchivePrivate = "private" // stored, readable through the API only
|
||||
ArchiveOff = "off" // not stored
|
||||
)
|
||||
|
||||
// Pending actions.
|
||||
const (
|
||||
ActionJoin = "join"
|
||||
|
|
@ -41,12 +48,13 @@ const (
|
|||
|
||||
// List is a mailing list.
|
||||
type List struct {
|
||||
ID int64
|
||||
Name string // local part, lowercase
|
||||
Domain string // lowercase
|
||||
Title string
|
||||
PostPolicy string
|
||||
JoinPolicy string
|
||||
ID int64
|
||||
Name string // local part, lowercase
|
||||
Domain string // lowercase
|
||||
Title string
|
||||
PostPolicy string
|
||||
JoinPolicy string
|
||||
ArchivePolicy string
|
||||
}
|
||||
|
||||
// Address returns the list posting address.
|
||||
|
|
@ -94,6 +102,7 @@ CREATE TABLE IF NOT EXISTS lists (
|
|||
title TEXT NOT NULL DEFAULT '',
|
||||
post_policy TEXT NOT NULL DEFAULT 'members',
|
||||
join_policy TEXT NOT NULL DEFAULT 'open',
|
||||
archive_policy TEXT NOT NULL DEFAULT 'private',
|
||||
UNIQUE(name, domain)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS members (
|
||||
|
|
@ -119,6 +128,30 @@ CREATE TABLE IF NOT EXISTS pending (
|
|||
action TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS archive (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
||||
thread_id INTEGER NOT NULL DEFAULT 0,
|
||||
message_id TEXT NOT NULL,
|
||||
in_reply_to TEXT NOT NULL DEFAULT '',
|
||||
subject TEXT NOT NULL DEFAULT '',
|
||||
sender TEXT NOT NULL DEFAULT '',
|
||||
date INTEGER NOT NULL,
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE(list_id, message_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS archive_list_date ON archive(list_id, date DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS archive_thread ON archive(list_id, thread_id);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5(
|
||||
subject, body, sender, content='archive', content_rowid='id'
|
||||
);
|
||||
CREATE TRIGGER IF NOT EXISTS archive_ai AFTER INSERT ON archive BEGIN
|
||||
INSERT INTO archive_fts(rowid, subject, body, sender) VALUES (new.id, new.subject, new.body, new.sender);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN
|
||||
INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender);
|
||||
END;
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
|
|
@ -180,6 +213,15 @@ func migrate(db *sql.DB) error {
|
|||
}
|
||||
}
|
||||
}
|
||||
if ok, err := hasColumn(db, "lists", "name"); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
if has, _ := hasColumn(db, "lists", "archive_policy"); !has {
|
||||
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN archive_policy TEXT NOT NULL DEFAULT 'private'`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err := db.Exec(schema)
|
||||
return err
|
||||
}
|
||||
|
|
@ -203,11 +245,11 @@ func affected(res sql.Result, err error) error {
|
|||
|
||||
// Lists.
|
||||
|
||||
const listCols = `id, name, domain, title, post_policy, join_policy`
|
||||
const listCols = `id, name, domain, title, post_policy, join_policy, archive_policy`
|
||||
|
||||
func scanList(row interface{ Scan(...any) error }) (List, error) {
|
||||
var l List
|
||||
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy)
|
||||
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy, &l.ArchivePolicy)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return List{}, ErrNotFound
|
||||
}
|
||||
|
|
@ -265,15 +307,18 @@ func (s *Store) UpdateList(id int64, title string) error {
|
|||
return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id))
|
||||
}
|
||||
|
||||
// SetPolicies changes the post and join policies of a list.
|
||||
func (s *Store) SetPolicies(id int64, post, join string) error {
|
||||
// SetPolicies changes the post, join and archive policies of a list.
|
||||
func (s *Store) SetPolicies(id int64, post, join, archive string) error {
|
||||
if post != PostMembers && post != PostModerated {
|
||||
return fmt.Errorf("invalid post policy %q", post)
|
||||
}
|
||||
if join != JoinOpen && join != JoinClosed {
|
||||
return fmt.Errorf("invalid join policy %q", join)
|
||||
}
|
||||
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=? WHERE id=?`, post, join, id))
|
||||
if archive != ArchivePublic && archive != ArchivePrivate && archive != ArchiveOff {
|
||||
return fmt.Errorf("invalid archive policy %q", archive)
|
||||
}
|
||||
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=?, archive_policy=? WHERE id=?`, post, join, archive, id))
|
||||
}
|
||||
|
||||
// DeleteList removes a list with its members, held messages and pending requests.
|
||||
|
|
@ -485,3 +530,159 @@ func (s *Store) Secret() ([]byte, error) {
|
|||
}
|
||||
return hex.DecodeString(v)
|
||||
}
|
||||
|
||||
// Archive.
|
||||
|
||||
// Message is an archived post.
|
||||
type Message struct {
|
||||
ID int64
|
||||
ListID int64
|
||||
ThreadID int64
|
||||
MessageID string // Message-ID header, unique per list
|
||||
InReplyTo string
|
||||
Subject string
|
||||
Sender string // display name of the original author
|
||||
Date time.Time
|
||||
Body string // plain text; empty in listings
|
||||
Size int
|
||||
}
|
||||
|
||||
const msgCols = `id, list_id, thread_id, message_id, in_reply_to, subject, sender, date, size`
|
||||
|
||||
func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, error) {
|
||||
var m Message
|
||||
var ts int64
|
||||
var err error
|
||||
if withBody {
|
||||
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size, &m.Body)
|
||||
} else {
|
||||
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size)
|
||||
}
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Message{}, ErrNotFound
|
||||
}
|
||||
m.Date = time.Unix(ts, 0)
|
||||
return m, err
|
||||
}
|
||||
|
||||
// ArchiveMessage stores a post. A message already archived in the same list
|
||||
// (same Message-ID) is not stored again: created is false and the existing id
|
||||
// is returned, so MTA retries are harmless.
|
||||
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
|
||||
if m.MessageID == "" {
|
||||
return 0, false, errors.New("message id is required")
|
||||
}
|
||||
err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
|
||||
if err == nil {
|
||||
return id, false, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, false, err
|
||||
}
|
||||
var thread int64
|
||||
if m.InReplyTo != "" {
|
||||
e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
|
||||
if e != nil && !errors.Is(e, sql.ErrNoRows) {
|
||||
return 0, false, e
|
||||
}
|
||||
}
|
||||
res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)`,
|
||||
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
id, _ = res.LastInsertId()
|
||||
if thread == 0 {
|
||||
if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
}
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
// GetMessage returns an archived message with its body.
|
||||
func (s *Store) GetMessage(id int64) (Message, error) {
|
||||
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)
|
||||
}
|
||||
|
||||
// DeleteMessage removes an archived message and its search entry.
|
||||
func (s *Store) DeleteMessage(id int64) error {
|
||||
return affected(s.db.Exec(`DELETE FROM archive WHERE id=?`, id))
|
||||
}
|
||||
|
||||
func (s *Store) queryMessages(q string, args ...any) ([]Message, error) {
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Message
|
||||
for rows.Next() {
|
||||
m, err := scanMessage(rows, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListMessages returns a page of a list's archive, newest first, and the total
|
||||
// count. Bodies are not loaded.
|
||||
func (s *Store) ListMessages(listID int64, limit, offset int) ([]Message, int, error) {
|
||||
var total int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive WHERE list_id=?`, listID).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out, err := s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? ORDER BY date DESC, id DESC LIMIT ? OFFSET ?`,
|
||||
listID, limit, offset)
|
||||
return out, total, err
|
||||
}
|
||||
|
||||
// ftsQuery turns free text into an FTS5 query where every word is a quoted
|
||||
// literal (AND semantics), so user input can never be read as FTS syntax.
|
||||
func ftsQuery(q string) string {
|
||||
words := strings.Fields(q)
|
||||
for i, w := range words {
|
||||
words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"`
|
||||
}
|
||||
return strings.Join(words, " ")
|
||||
}
|
||||
|
||||
// SearchMessages runs a full-text search over subject, body and sender of one
|
||||
// list, newest first. Bodies are not loaded.
|
||||
func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([]Message, int, error) {
|
||||
match := ftsQuery(query)
|
||||
if match == "" {
|
||||
return nil, 0, nil
|
||||
}
|
||||
var total int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
|
||||
WHERE archive_fts MATCH ? AND a.list_id=?`, match, listID).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out, err := s.queryMessages(`SELECT a.id, a.list_id, a.thread_id, a.message_id, a.in_reply_to, a.subject, a.sender, a.date, a.size
|
||||
FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
|
||||
WHERE archive_fts MATCH ? AND a.list_id=? ORDER BY a.date DESC, a.id DESC LIMIT ? OFFSET ?`,
|
||||
match, listID, limit, offset)
|
||||
return out, total, err
|
||||
}
|
||||
|
||||
// ThreadMessages returns every message of a thread, oldest first, with bodies.
|
||||
func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) {
|
||||
rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Message
|
||||
for rows.Next() {
|
||||
m, err := scanMessage(rows, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue