From d8275aaa1f22e339418319103a699a50d19f8dce Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 15:17:13 +0100 Subject: [PATCH] feat(store,api): archive with full-text search, threading and archive policy - Add archive table with FTS5 index kept in sync by triggers - Deduplicate by Message-ID per list so MTA retries do not repeat posts - Thread messages through In-Reply-To within the same list - Quote every search word so user input is never FTS syntax - Add archive_policy (public, private, off) with migration - Add API to list, search, read and delete archived messages --- internal/api/api.go | 120 ++++++++++++++++- internal/api/api_test.go | 117 ++++++++++++++++- internal/ingest/ingest_test.go | 2 +- internal/router/m2_test.go | 8 +- internal/store/archive_test.go | 229 +++++++++++++++++++++++++++++++++ internal/store/m2_test.go | 6 +- internal/store/store.go | 223 ++++++++++++++++++++++++++++++-- 7 files changed, 672 insertions(+), 33 deletions(-) create mode 100644 internal/store/archive_test.go diff --git a/internal/api/api.go b/internal/api/api.go index 1d027bf..f4d2e8a 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -45,6 +45,9 @@ func New(s *store.Store, token string, rel Releaser) http.Handler { mux.Handle("GET /api/v1/lists/{id}/members", srv.auth(srv.listMembers)) mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember)) mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember)) + mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive)) + mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived)) + mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived)) mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld)) mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld)) mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld)) @@ -72,13 +75,14 @@ type listDTO struct { Title string `json:"title"` Address string `json:"address"` - PostPolicy string `json:"post_policy"` - JoinPolicy string `json:"join_policy"` + PostPolicy string `json:"post_policy"` + JoinPolicy string `json:"join_policy"` + ArchivePolicy string `json:"archive_policy"` } func toListDTO(l store.List) listDTO { return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address(), - PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy} + PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy, ArchivePolicy: l.ArchivePolicy} } type memberDTO struct { @@ -195,11 +199,13 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { Title *string `json:"title"` PostPolicy *string `json:"post_policy"` JoinPolicy *string `json:"join_policy"` + + ArchivePolicy *string `json:"archive_policy"` } if !decodeBody(w, r, &in) { return } - if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil { + if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil && in.ArchivePolicy == nil { writeError(w, http.StatusBadRequest, "nothing to update") return } @@ -222,17 +228,21 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { if in.JoinPolicy != nil { l.JoinPolicy = *in.JoinPolicy } + if in.ArchivePolicy != nil { + l.ArchivePolicy = *in.ArchivePolicy + } // Validate before writing anything. if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated) || - (l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) { - writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy must be open or closed") + (l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) || + (l.ArchivePolicy != store.ArchivePublic && l.ArchivePolicy != store.ArchivePrivate && l.ArchivePolicy != store.ArchiveOff) { + writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy open or closed; archive_policy public, private or off") return } if err := s.store.UpdateList(id, l.Title); err != nil { storeError(w, err) return } - if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy); err != nil { + if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy, l.ArchivePolicy); err != nil { storeError(w, err) return } @@ -385,3 +395,99 @@ func (s *server) discardHeld(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(http.StatusNoContent) } + +// Handlers: archive. + +const ( + defaultPageSize = 50 + maxPageSize = 200 +) + +type messageDTO struct { + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + MessageID string `json:"message_id"` + InReplyTo string `json:"in_reply_to,omitempty"` + Subject string `json:"subject"` + Sender string `json:"sender"` + Date time.Time `json:"date"` + Size int `json:"size"` + Body string `json:"body,omitempty"` +} + +func toMessageDTO(m store.Message) messageDTO { + return messageDTO{ID: m.ID, ThreadID: m.ThreadID, MessageID: m.MessageID, InReplyTo: m.InReplyTo, + Subject: m.Subject, Sender: m.Sender, Date: m.Date, Size: m.Size, Body: m.Body} +} + +// intParam reads a non-negative integer query parameter. +func intParam(r *http.Request, name string, def int) (int, bool) { + v := r.URL.Query().Get(name) + if v == "" { + return def, true + } + n, err := strconv.Atoi(v) + return n, err == nil && n >= 0 +} + +func (s *server) listArchive(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + limit, ok1 := intParam(r, "limit", defaultPageSize) + offset, ok2 := intParam(r, "offset", 0) + if !ok1 || !ok2 { + writeError(w, http.StatusBadRequest, "limit and offset must be non-negative integers") + return + } + limit = min(max(limit, 1), maxPageSize) + if _, err := s.store.GetList(id); err != nil { + storeError(w, err) + return + } + var ( + ms []store.Message + total int + err error + ) + if q := strings.TrimSpace(r.URL.Query().Get("q")); q != "" { + ms, total, err = s.store.SearchMessages(id, q, limit, offset) + } else { + ms, total, err = s.store.ListMessages(id, limit, offset) + } + if err != nil { + storeError(w, err) + return + } + out := make([]messageDTO, 0, len(ms)) + for _, m := range ms { + out = append(out, toMessageDTO(m)) + } + writeJSON(w, http.StatusOK, map[string]any{"total": total, "messages": out}) +} + +func (s *server) getArchived(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + m, err := s.store.GetMessage(id) + if err != nil { + storeError(w, err) + return + } + writeJSON(w, http.StatusOK, toMessageDTO(m)) +} + +func (s *server) deleteArchived(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + if err := s.store.DeleteMessage(id); err != nil { + storeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 92382a7..e7f6da9 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -3,10 +3,12 @@ package api import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" "testing" + "time" "codeberg.org/squidd/parsimony/internal/store" ) @@ -78,13 +80,14 @@ func TestEmptyTokenRejectsEverything(t *testing.T) { } type listJSON struct { - ID int64 `json:"id"` - Name string `json:"name"` - Domain string `json:"domain"` - Title string `json:"title"` - Address string `json:"address"` - PostPolicy string `json:"post_policy"` - JoinPolicy string `json:"join_policy"` + ID int64 `json:"id"` + Name string `json:"name"` + Domain string `json:"domain"` + Title string `json:"title"` + Address string `json:"address"` + PostPolicy string `json:"post_policy"` + JoinPolicy string `json:"join_policy"` + ArchivePolicy string `json:"archive_policy"` } func createList(t *testing.T, h http.Handler) listJSON { @@ -315,3 +318,103 @@ func TestApproveUnknownHeld(t *testing.T) { type errReleaser struct{ err error } func (e errReleaser) Release(context.Context, int64) error { return e.err } + +func archiveServer(t *testing.T) http.Handler { + t.Helper() + s, _ := store.Open(":memory:") + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i, subj := range []string{"First post", "Second post", "Lunch"} { + s.ArchiveMessage(l.ID, store.Message{ + MessageID: fmt.Sprintf("<%d@x>", i), Subject: subj, Sender: "Alice", + Date: time.Unix(1_700_000_000+int64(i), 0), Body: "body of " + subj, + }) + } + return New(s, token, nil) +} + +type archiveJSON struct { + Total int `json:"total"` + Messages []struct { + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + Subject string `json:"subject"` + Sender string `json:"sender"` + Body string `json:"body"` + } `json:"messages"` +} + +func TestArchivePolicyViaAPI(t *testing.T) { + h := newServer(t) + l := createList(t, h) + if l.ArchivePolicy != "private" { + t.Fatalf("default = %q", l.ArchivePolicy) + } + rec := do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"public"}`, true) + expect(t, rec, 200) + var u listJSON + decode(t, rec, &u) + if u.ArchivePolicy != "public" || u.PostPolicy != "members" { + t.Fatalf("patched: %+v", u) + } + expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"bogus"}`, true), 400) +} + +func TestArchiveList(t *testing.T) { + h := archiveServer(t) + rec := do(t, h, "GET", "/api/v1/lists/1/archive", "", true) + expect(t, rec, 200) + var out archiveJSON + decode(t, rec, &out) + if out.Total != 3 || len(out.Messages) != 3 || out.Messages[0].Subject != "Lunch" { + t.Fatalf("archive: %+v", out) + } + if out.Messages[0].Body != "" { + t.Fatal("listings must not include bodies") + } + + rec = do(t, h, "GET", "/api/v1/lists/1/archive?limit=2&offset=2", "", true) + decode(t, rec, &out) + if out.Total != 3 || len(out.Messages) != 1 || out.Messages[0].Subject != "First post" { + t.Fatalf("page: %+v", out) + } + + rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=post", "", true) + decode(t, rec, &out) + if out.Total != 2 || len(out.Messages) != 2 { + t.Fatalf("search: %+v", out) + } + rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=nothingmatches", "", true) + decode(t, rec, &out) + if out.Total != 0 || out.Messages == nil || len(out.Messages) != 0 { + t.Fatalf("empty search must be an empty array: %+v", out) + } + expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=abc", "", true), 400) + expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?offset=-1", "", true), 400) + expect(t, do(t, h, "GET", "/api/v1/lists/99/archive", "", true), 404) + expect(t, do(t, h, "GET", "/api/v1/lists/1/archive", "", false), 401) +} + +func TestArchiveLimitIsCapped(t *testing.T) { + h := archiveServer(t) + expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=100000", "", true), 200) +} + +func TestArchiveGetAndDelete(t *testing.T) { + h := archiveServer(t) + rec := do(t, h, "GET", "/api/v1/archive/1", "", true) + expect(t, rec, 200) + var m struct { + Subject string `json:"subject"` + Body string `json:"body"` + } + decode(t, rec, &m) + if m.Subject != "First post" || m.Body != "body of First post" { + t.Fatalf("message: %+v", m) + } + expect(t, do(t, h, "GET", "/api/v1/archive/99", "", true), 404) + expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 204) + expect(t, do(t, h, "GET", "/api/v1/archive/1", "", true), 404) + expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 404) + expect(t, do(t, h, "DELETE", "/api/v1/archive/2", "", false), 401) +} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 051c042..4535268 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -180,7 +180,7 @@ func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) { t.Cleanup(func() { s.Close() }) l, _ := s.CreateList("dev", "example.org", "Dev") s.AddMember(l.ID, "alice@example.com", "Alice") - s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen) + s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) relay := &outbound.Fake{} srv := NewServer(router.New(s), relay) ln, _ := net.Listen("tcp", "127.0.0.1:0") diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index 48caf8f..c1b0e20 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -85,7 +85,7 @@ func TestDisabledMembersAreSkipped(t *testing.T) { func TestModeratedListHoldsPosts(t *testing.T) { e := newEnv(t) - e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen) + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) for _, sender := range []string{"alice@example.com", "stranger@x.com"} { _, err := e.r.Route(sender, "dev@example.org", []byte(sample)) if !errors.Is(err, ErrHeld) { @@ -103,7 +103,7 @@ func TestModeratedListHoldsPosts(t *testing.T) { func TestReleaseHeld(t *testing.T) { e := newEnv(t) - e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen) + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) e.r.Route("stranger@x.com", "dev@example.org", []byte(sample)) held, _ := e.s.ListHeld(e.list.ID) @@ -182,7 +182,7 @@ func TestJoinRateLimitedAndSilentForMembers(t *testing.T) { func TestJoinClosed(t *testing.T) { e := newEnv(t) - e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) if !errors.Is(err, ErrJoinClosed) { t.Fatalf("want ErrJoinClosed, got %v", err) @@ -196,7 +196,7 @@ func TestConfirmAfterListClosed(t *testing.T) { e := newEnv(t) e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) tok := token(t, lastMail(t, e.out)) - e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) if !errors.Is(err, ErrJoinClosed) { t.Fatalf("want ErrJoinClosed, got %v", err) diff --git a/internal/store/archive_test.go b/internal/store/archive_test.go new file mode 100644 index 0000000..7fe6b01 --- /dev/null +++ b/internal/store/archive_test.go @@ -0,0 +1,229 @@ +package store + +import ( + "errors" + "strings" + "testing" + "time" +) + +func msg(id, subject, body string) Message { + return Message{MessageID: id, Subject: subject, Sender: "Alice", Date: time.Unix(1_700_000_000, 0), Body: body} +} + +func TestArchivePolicyDefaultsAndUpdate(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if l.ArchivePolicy != ArchivePrivate { + t.Fatalf("default = %q", l.ArchivePolicy) + } + if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, ArchivePublic); err != nil { + t.Fatal(err) + } + got, _ := s.GetList(l.ID) + if got.ArchivePolicy != ArchivePublic { + t.Fatalf("policy = %q", got.ArchivePolicy) + } + if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, "bogus"); err == nil { + t.Fatal("invalid archive policy must be rejected") + } +} + +func TestArchiveAndFetch(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + m := msg("<1@x>", "Hello", "first body") + id, created, err := s.ArchiveMessage(l.ID, m) + if err != nil || !created || id == 0 { + t.Fatalf("archive: %d %v %v", id, created, err) + } + got, err := s.GetMessage(id) + if err != nil { + t.Fatal(err) + } + if got.Subject != "Hello" || got.Body != "first body" || got.ListID != l.ID || got.ThreadID != id || got.Size != len("first body") { + t.Fatalf("got %+v", got) + } + if !got.Date.Equal(m.Date) { + t.Fatalf("date = %v", got.Date) + } + if _, err := s.GetMessage(999); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestArchiveDeduplicatesByMessageID(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + id1, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a")) + id2, created, err := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a")) + if err != nil || created || id2 != id1 { + t.Fatalf("retry must not duplicate: %d %v %v", id2, created, err) + } + other, _ := s.CreateList("ops", "example.org", "Ops") + if _, created, _ := s.ArchiveMessage(other.ID, msg("<1@x>", "Hello", "a")); !created { + t.Fatal("same Message-ID in another list is a different message") + } +} + +func TestThreading(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + root, _, _ := s.ArchiveMessage(l.ID, msg("", "Topic", "r")) + reply := msg("", "Re: Topic", "r1") + reply.InReplyTo = "" + r1, _, _ := s.ArchiveMessage(l.ID, reply) + nested := msg("", "Re: Topic", "r2") + nested.InReplyTo = "" + r2, _, _ := s.ArchiveMessage(l.ID, nested) + orphan := msg("", "Other", "o") + orphan.InReplyTo = "" + o, _, _ := s.ArchiveMessage(l.ID, orphan) + + for _, id := range []int64{r1, r2} { + m, _ := s.GetMessage(id) + if m.ThreadID != root { + t.Fatalf("message %d thread = %d, want %d", id, m.ThreadID, root) + } + } + if m, _ := s.GetMessage(o); m.ThreadID != o { + t.Fatalf("orphan must start its own thread, got %d", m.ThreadID) + } + th, err := s.ThreadMessages(l.ID, root) + if err != nil || len(th) != 3 { + t.Fatalf("thread: %d %v", len(th), err) + } +} + +func TestThreadingIgnoresOtherLists(t *testing.T) { + s := newTestStore(t) + a, _ := s.CreateList("a", "example.org", "A") + b, _ := s.CreateList("b", "example.org", "B") + s.ArchiveMessage(a.ID, msg("", "Topic", "r")) + reply := msg("", "Re: Topic", "r") + reply.InReplyTo = "" + id, _, _ := s.ArchiveMessage(b.ID, reply) + if m, _ := s.GetMessage(id); m.ThreadID != id { + t.Fatal("a parent in another list must not be used") + } +} + +func TestListMessagesNewestFirstWithPaging(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i := 0; i < 5; i++ { + m := msg("", "S"+string(rune('a'+i)), "body") + m.Date = time.Unix(1_700_000_000+int64(i), 0) + s.ArchiveMessage(l.ID, m) + } + got, total, err := s.ListMessages(l.ID, 2, 0) + if err != nil || total != 5 || len(got) != 2 { + t.Fatalf("page 1: %d %d %v", len(got), total, err) + } + if got[0].Subject != "Se" || got[1].Subject != "Sd" { + t.Fatalf("order: %s %s", got[0].Subject, got[1].Subject) + } + if got[0].Body != "" { + t.Fatal("listings must not load bodies") + } + got, _, _ = s.ListMessages(l.ID, 2, 4) + if len(got) != 1 || got[0].Subject != "Sa" { + t.Fatalf("last page: %+v", got) + } +} + +func TestSearch(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + other, _ := s.CreateList("ops", "example.org", "Ops") + s.ArchiveMessage(l.ID, msg("<1@x>", "Deploy failed", "the kubernetes rollout broke")) + s.ArchiveMessage(l.ID, msg("<2@x>", "Lunch", "pizza on friday")) + s.ArchiveMessage(other.ID, msg("<3@x>", "Deploy ok", "kubernetes is fine here")) + + got, total, err := s.SearchMessages(l.ID, "kubernetes", 10, 0) + if err != nil || total != 1 || len(got) != 1 || got[0].Subject != "Deploy failed" { + t.Fatalf("search: %+v %d %v", got, total, err) + } + // Subject, sender and multiple terms (AND). + if got, _, _ := s.SearchMessages(l.ID, "lunch pizza", 10, 0); len(got) != 1 { + t.Fatalf("subject+body: %+v", got) + } + if got, _, _ := s.SearchMessages(l.ID, "alice", 10, 0); len(got) != 2 { + t.Fatalf("sender: %+v", got) + } + if got, _, _ := s.SearchMessages(l.ID, "lunch kubernetes", 10, 0); len(got) != 0 { + t.Fatalf("AND semantics: %+v", got) + } +} + +func TestSearchIsSafeAgainstSyntax(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "world")) + for _, q := range []string{`"`, `foo AND`, `NEAR(`, `*`, `a OR b`, `col:x`, `)(`, "", " "} { + if _, _, err := s.SearchMessages(l.ID, q, 10, 0); err != nil { + t.Errorf("query %q must not error: %v", q, err) + } + } + // FTS operators are plain words: "world OR zzz" needs both terms. + if got, _, _ := s.SearchMessages(l.ID, "world OR zzz", 10, 0); len(got) != 0 { + t.Fatalf("operators must be literal: %+v", got) + } +} + +func TestDeleteMessageRemovesFromSearch(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + id, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Secret", "leaked password")) + if err := s.DeleteMessage(id); err != nil { + t.Fatal(err) + } + if got, _, _ := s.SearchMessages(l.ID, "password", 10, 0); len(got) != 0 { + t.Fatalf("deleted message still searchable: %+v", got) + } + if err := s.DeleteMessage(id); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestDeleteListCleansArchiveAndIndex(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "uniqueword")) + if err := s.DeleteList(l.ID); err != nil { + t.Fatal(err) + } + var n int + s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts WHERE archive_fts MATCH 'uniqueword'`).Scan(&n) + if n != 0 { + t.Fatal("search index must follow cascade deletes") + } +} + +func TestMigrationAddsArchiveColumn(t *testing.T) { + // A database from milestone 2 has no archive_policy column. + path := t.TempDir() + "/m2.db" + s, _ := Open(path) + s.CreateList("dev", "example.org", "Dev") + s.db.Exec(`ALTER TABLE lists DROP COLUMN archive_policy`) + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + l, err := s2.FindList("dev@example.org") + if err != nil || l.ArchivePolicy != ArchivePrivate { + t.Fatalf("list: %+v %v", l, err) + } +} + +func TestLongBodyIsStoredAsIs(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + body := strings.Repeat("x", 100_000) + id, _, _ := s.ArchiveMessage(l.ID, msg("", "Big", body)) + if got, _ := s.GetMessage(id); len(got.Body) != len(body) { + t.Fatalf("body len %d", len(got.Body)) + } +} diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 0e49708..1ca622d 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -14,17 +14,17 @@ func TestListPolicies(t *testing.T) { if l.PostPolicy != "members" || l.JoinPolicy != "open" { t.Fatalf("defaults: %+v", l) } - if err := s.SetPolicies(l.ID, "moderated", "closed"); err != nil { + if err := s.SetPolicies(l.ID, "moderated", "closed", "private"); err != nil { t.Fatal(err) } got, _ := s.GetList(l.ID) if got.PostPolicy != "moderated" || got.JoinPolicy != "closed" { t.Fatalf("policies: %+v", got) } - if err := s.SetPolicies(l.ID, "bogus", "open"); err == nil { + if err := s.SetPolicies(l.ID, "bogus", "open", "private"); err == nil { t.Fatal("invalid policy must be rejected") } - if err := s.SetPolicies(999, "members", "open"); !errors.Is(err, ErrNotFound) { + if err := s.SetPolicies(999, "members", "open", "private"); !errors.Is(err, ErrNotFound) { t.Fatalf("want ErrNotFound, got %v", err) } } diff --git a/internal/store/store.go b/internal/store/store.go index e74e487..e1ea6fd 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -28,6 +28,13 @@ const ( JoinClosed = "closed" // only the admin adds members ) +// Archive policy values. +const ( + ArchivePublic = "public" // browsable on the web + ArchivePrivate = "private" // stored, readable through the API only + ArchiveOff = "off" // not stored +) + // Pending actions. const ( ActionJoin = "join" @@ -41,12 +48,13 @@ const ( // List is a mailing list. type List struct { - ID int64 - Name string // local part, lowercase - Domain string // lowercase - Title string - PostPolicy string - JoinPolicy string + ID int64 + Name string // local part, lowercase + Domain string // lowercase + Title string + PostPolicy string + JoinPolicy string + ArchivePolicy string } // Address returns the list posting address. @@ -94,6 +102,7 @@ CREATE TABLE IF NOT EXISTS lists ( title TEXT NOT NULL DEFAULT '', post_policy TEXT NOT NULL DEFAULT 'members', join_policy TEXT NOT NULL DEFAULT 'open', + archive_policy TEXT NOT NULL DEFAULT 'private', UNIQUE(name, domain) ); CREATE TABLE IF NOT EXISTS members ( @@ -119,6 +128,30 @@ CREATE TABLE IF NOT EXISTS pending ( action TEXT NOT NULL, created_at INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS archive ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + thread_id INTEGER NOT NULL DEFAULT 0, + message_id TEXT NOT NULL, + in_reply_to TEXT NOT NULL DEFAULT '', + subject TEXT NOT NULL DEFAULT '', + sender TEXT NOT NULL DEFAULT '', + date INTEGER NOT NULL, + body TEXT NOT NULL DEFAULT '', + size INTEGER NOT NULL DEFAULT 0, + UNIQUE(list_id, message_id) +); +CREATE INDEX IF NOT EXISTS archive_list_date ON archive(list_id, date DESC, id DESC); +CREATE INDEX IF NOT EXISTS archive_thread ON archive(list_id, thread_id); +CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5( + subject, body, sender, content='archive', content_rowid='id' +); +CREATE TRIGGER IF NOT EXISTS archive_ai AFTER INSERT ON archive BEGIN + INSERT INTO archive_fts(rowid, subject, body, sender) VALUES (new.id, new.subject, new.body, new.sender); +END; +CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN + INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender); +END; CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -180,6 +213,15 @@ func migrate(db *sql.DB) error { } } } + if ok, err := hasColumn(db, "lists", "name"); err != nil { + return err + } else if ok { + if has, _ := hasColumn(db, "lists", "archive_policy"); !has { + if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN archive_policy TEXT NOT NULL DEFAULT 'private'`); err != nil { + return err + } + } + } _, err := db.Exec(schema) return err } @@ -203,11 +245,11 @@ func affected(res sql.Result, err error) error { // Lists. -const listCols = `id, name, domain, title, post_policy, join_policy` +const listCols = `id, name, domain, title, post_policy, join_policy, archive_policy` func scanList(row interface{ Scan(...any) error }) (List, error) { var l List - err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy) + err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy, &l.ArchivePolicy) if errors.Is(err, sql.ErrNoRows) { return List{}, ErrNotFound } @@ -265,15 +307,18 @@ func (s *Store) UpdateList(id int64, title string) error { return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id)) } -// SetPolicies changes the post and join policies of a list. -func (s *Store) SetPolicies(id int64, post, join string) error { +// SetPolicies changes the post, join and archive policies of a list. +func (s *Store) SetPolicies(id int64, post, join, archive string) error { if post != PostMembers && post != PostModerated { return fmt.Errorf("invalid post policy %q", post) } if join != JoinOpen && join != JoinClosed { return fmt.Errorf("invalid join policy %q", join) } - return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=? WHERE id=?`, post, join, id)) + if archive != ArchivePublic && archive != ArchivePrivate && archive != ArchiveOff { + return fmt.Errorf("invalid archive policy %q", archive) + } + return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=?, archive_policy=? WHERE id=?`, post, join, archive, id)) } // DeleteList removes a list with its members, held messages and pending requests. @@ -485,3 +530,159 @@ func (s *Store) Secret() ([]byte, error) { } return hex.DecodeString(v) } + +// Archive. + +// Message is an archived post. +type Message struct { + ID int64 + ListID int64 + ThreadID int64 + MessageID string // Message-ID header, unique per list + InReplyTo string + Subject string + Sender string // display name of the original author + Date time.Time + Body string // plain text; empty in listings + Size int +} + +const msgCols = `id, list_id, thread_id, message_id, in_reply_to, subject, sender, date, size` + +func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, error) { + var m Message + var ts int64 + var err error + if withBody { + err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size, &m.Body) + } else { + err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size) + } + if errors.Is(err, sql.ErrNoRows) { + return Message{}, ErrNotFound + } + m.Date = time.Unix(ts, 0) + return m, err +} + +// ArchiveMessage stores a post. A message already archived in the same list +// (same Message-ID) is not stored again: created is false and the existing id +// is returned, so MTA retries are harmless. +func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) { + if m.MessageID == "" { + return 0, false, errors.New("message id is required") + } + err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id) + if err == nil { + return id, false, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, false, err + } + var thread int64 + if m.InReplyTo != "" { + e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread) + if e != nil && !errors.Is(e, sql.ErrNoRows) { + return 0, false, e + } + } + res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size) + VALUES(?,?,?,?,?,?,?,?,?)`, + listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body)) + if err != nil { + return 0, false, err + } + id, _ = res.LastInsertId() + if thread == 0 { + if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil { + return 0, false, err + } + } + return id, true, nil +} + +// GetMessage returns an archived message with its body. +func (s *Store) GetMessage(id int64) (Message, error) { + return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true) +} + +// DeleteMessage removes an archived message and its search entry. +func (s *Store) DeleteMessage(id int64) error { + return affected(s.db.Exec(`DELETE FROM archive WHERE id=?`, id)) +} + +func (s *Store) queryMessages(q string, args ...any) ([]Message, error) { + rows, err := s.db.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Message + for rows.Next() { + m, err := scanMessage(rows, false) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} + +// ListMessages returns a page of a list's archive, newest first, and the total +// count. Bodies are not loaded. +func (s *Store) ListMessages(listID int64, limit, offset int) ([]Message, int, error) { + var total int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive WHERE list_id=?`, listID).Scan(&total); err != nil { + return nil, 0, err + } + out, err := s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? ORDER BY date DESC, id DESC LIMIT ? OFFSET ?`, + listID, limit, offset) + return out, total, err +} + +// ftsQuery turns free text into an FTS5 query where every word is a quoted +// literal (AND semantics), so user input can never be read as FTS syntax. +func ftsQuery(q string) string { + words := strings.Fields(q) + for i, w := range words { + words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"` + } + return strings.Join(words, " ") +} + +// SearchMessages runs a full-text search over subject, body and sender of one +// list, newest first. Bodies are not loaded. +func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([]Message, int, error) { + match := ftsQuery(query) + if match == "" { + return nil, 0, nil + } + var total int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid + WHERE archive_fts MATCH ? AND a.list_id=?`, match, listID).Scan(&total); err != nil { + return nil, 0, err + } + out, err := s.queryMessages(`SELECT a.id, a.list_id, a.thread_id, a.message_id, a.in_reply_to, a.subject, a.sender, a.date, a.size + FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid + WHERE archive_fts MATCH ? AND a.list_id=? ORDER BY a.date DESC, a.id DESC LIMIT ? OFFSET ?`, + match, listID, limit, offset) + return out, total, err +} + +// ThreadMessages returns every message of a thread, oldest first, with bodies. +func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) { + rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Message + for rows.Next() { + m, err := scanMessage(rows, true) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +}