diff --git a/README.md b/README.md index f3ab39e..2d989c8 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ A modern, single-binary mailing list manager: an evolution of Mailman. send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. - Moderated lists: posts wait in a queue that the API can approve or discard. - VERP bounces: every recipient gets a signed return address; 3 hard bounces disable a member. +- Archive with full-text search (SQLite FTS5) and threading. Each list has an + `archive_policy`: `private` (default, API only), `public` (web pages) or `off`. + Public pages hide email addresses. +- Per-recipient one-click unsubscribe links (RFC 8058) with signed tokens. - SQLite storage, one YAML config file. - Admin REST API under `/api/v1` (see `docs/openapi.yaml`). @@ -21,6 +25,17 @@ PARSIMONY_API_TOKEN= go run ./cmd/parsimony -config parsimony.yaml The API is disabled when `PARSIMONY_API_TOKEN` is not set. +Three listeners, each configurable: + +| Listener | Default | Purpose | +|----------|---------|---------| +| `lmtp` | `127.0.0.1:2424` | mail from Postfix | +| `api` | `127.0.0.1:8080` | admin REST API, bearer token | +| `web` | `127.0.0.1:8081` | public archive and unsubscribe pages (`""` disables) | + +Put the `web` listener behind a reverse proxy with TLS and set `base_url` to its public URL: +the unsubscribe links in every message use it. Keep the `api` listener private. + ``` curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists ``` diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 16c2b82..e5d16b8 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -22,6 +22,19 @@ components: address: {type: string} post_policy: {type: string, enum: [members, moderated]} join_policy: {type: string, enum: [open, closed]} + archive_policy: {type: string, enum: [public, private, off], description: "public: browsable on the web; private: API only (default); off: not stored"} + Message: + type: object + properties: + id: {type: integer} + thread_id: {type: integer} + message_id: {type: string} + in_reply_to: {type: string} + subject: {type: string} + sender: {type: string, description: Display name of the author} + date: {type: string, format: date-time} + size: {type: integer} + body: {type: string, description: Only when reading a single message} Member: type: object properties: @@ -108,6 +121,7 @@ paths: title: {type: string} post_policy: {type: string, enum: [members, moderated]} join_policy: {type: string, enum: [open, closed]} + archive_policy: {type: string, enum: [public, private, off]} responses: '200': description: OK @@ -191,3 +205,41 @@ paths: responses: '204': {description: Discarded} '404': {$ref: '#/components/responses/Error'} + /lists/{id}/archive: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + get: + summary: List or search archived messages, newest first + parameters: + - {name: q, in: query, schema: {type: string}, description: Full-text search over subject, body and sender; words are ANDed} + - {name: limit, in: query, schema: {type: integer, default: 50, maximum: 200}} + - {name: offset, in: query, schema: {type: integer, default: 0}} + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + total: {type: integer} + messages: {type: array, items: {$ref: '#/components/schemas/Message'}} + '400': {$ref: '#/components/responses/Error'} + '404': {$ref: '#/components/responses/Error'} + /archive/{id}: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + get: + summary: Read an archived message with its body + responses: + '200': + description: OK + content: + application/json: + schema: {$ref: '#/components/schemas/Message'} + '404': {$ref: '#/components/responses/Error'} + delete: + summary: Delete an archived message and its search entry + responses: + '204': {description: Deleted} + '404': {$ref: '#/components/responses/Error'} diff --git a/go.mod b/go.mod index 42eda6d..162f38c 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/ncruces/go-strftime v1.1.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.14.0 // indirect modernc.org/libc v1.80.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect diff --git a/go.sum b/go.sum index 022429f..e916675 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/internal/api/api.go b/internal/api/api.go index 1d027bf..f4d2e8a 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -45,6 +45,9 @@ func New(s *store.Store, token string, rel Releaser) http.Handler { mux.Handle("GET /api/v1/lists/{id}/members", srv.auth(srv.listMembers)) mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember)) mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember)) + mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive)) + mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived)) + mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived)) mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld)) mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld)) mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld)) @@ -72,13 +75,14 @@ type listDTO struct { Title string `json:"title"` Address string `json:"address"` - PostPolicy string `json:"post_policy"` - JoinPolicy string `json:"join_policy"` + PostPolicy string `json:"post_policy"` + JoinPolicy string `json:"join_policy"` + ArchivePolicy string `json:"archive_policy"` } func toListDTO(l store.List) listDTO { return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address(), - PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy} + PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy, ArchivePolicy: l.ArchivePolicy} } type memberDTO struct { @@ -195,11 +199,13 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { Title *string `json:"title"` PostPolicy *string `json:"post_policy"` JoinPolicy *string `json:"join_policy"` + + ArchivePolicy *string `json:"archive_policy"` } if !decodeBody(w, r, &in) { return } - if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil { + if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil && in.ArchivePolicy == nil { writeError(w, http.StatusBadRequest, "nothing to update") return } @@ -222,17 +228,21 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { if in.JoinPolicy != nil { l.JoinPolicy = *in.JoinPolicy } + if in.ArchivePolicy != nil { + l.ArchivePolicy = *in.ArchivePolicy + } // Validate before writing anything. if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated) || - (l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) { - writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy must be open or closed") + (l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) || + (l.ArchivePolicy != store.ArchivePublic && l.ArchivePolicy != store.ArchivePrivate && l.ArchivePolicy != store.ArchiveOff) { + writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy open or closed; archive_policy public, private or off") return } if err := s.store.UpdateList(id, l.Title); err != nil { storeError(w, err) return } - if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy); err != nil { + if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy, l.ArchivePolicy); err != nil { storeError(w, err) return } @@ -385,3 +395,99 @@ func (s *server) discardHeld(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(http.StatusNoContent) } + +// Handlers: archive. + +const ( + defaultPageSize = 50 + maxPageSize = 200 +) + +type messageDTO struct { + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + MessageID string `json:"message_id"` + InReplyTo string `json:"in_reply_to,omitempty"` + Subject string `json:"subject"` + Sender string `json:"sender"` + Date time.Time `json:"date"` + Size int `json:"size"` + Body string `json:"body,omitempty"` +} + +func toMessageDTO(m store.Message) messageDTO { + return messageDTO{ID: m.ID, ThreadID: m.ThreadID, MessageID: m.MessageID, InReplyTo: m.InReplyTo, + Subject: m.Subject, Sender: m.Sender, Date: m.Date, Size: m.Size, Body: m.Body} +} + +// intParam reads a non-negative integer query parameter. +func intParam(r *http.Request, name string, def int) (int, bool) { + v := r.URL.Query().Get(name) + if v == "" { + return def, true + } + n, err := strconv.Atoi(v) + return n, err == nil && n >= 0 +} + +func (s *server) listArchive(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + limit, ok1 := intParam(r, "limit", defaultPageSize) + offset, ok2 := intParam(r, "offset", 0) + if !ok1 || !ok2 { + writeError(w, http.StatusBadRequest, "limit and offset must be non-negative integers") + return + } + limit = min(max(limit, 1), maxPageSize) + if _, err := s.store.GetList(id); err != nil { + storeError(w, err) + return + } + var ( + ms []store.Message + total int + err error + ) + if q := strings.TrimSpace(r.URL.Query().Get("q")); q != "" { + ms, total, err = s.store.SearchMessages(id, q, limit, offset) + } else { + ms, total, err = s.store.ListMessages(id, limit, offset) + } + if err != nil { + storeError(w, err) + return + } + out := make([]messageDTO, 0, len(ms)) + for _, m := range ms { + out = append(out, toMessageDTO(m)) + } + writeJSON(w, http.StatusOK, map[string]any{"total": total, "messages": out}) +} + +func (s *server) getArchived(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + m, err := s.store.GetMessage(id) + if err != nil { + storeError(w, err) + return + } + writeJSON(w, http.StatusOK, toMessageDTO(m)) +} + +func (s *server) deleteArchived(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + if err := s.store.DeleteMessage(id); err != nil { + storeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 92382a7..0045542 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -3,10 +3,12 @@ package api import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" "testing" + "time" "codeberg.org/squidd/parsimony/internal/store" ) @@ -78,13 +80,14 @@ func TestEmptyTokenRejectsEverything(t *testing.T) { } type listJSON struct { - ID int64 `json:"id"` - Name string `json:"name"` - Domain string `json:"domain"` - Title string `json:"title"` - Address string `json:"address"` - PostPolicy string `json:"post_policy"` - JoinPolicy string `json:"join_policy"` + ID int64 `json:"id"` + Name string `json:"name"` + Domain string `json:"domain"` + Title string `json:"title"` + Address string `json:"address"` + PostPolicy string `json:"post_policy"` + JoinPolicy string `json:"join_policy"` + ArchivePolicy string `json:"archive_policy"` } func createList(t *testing.T, h http.Handler) listJSON { @@ -315,3 +318,120 @@ func TestApproveUnknownHeld(t *testing.T) { type errReleaser struct{ err error } func (e errReleaser) Release(context.Context, int64) error { return e.err } + +func archiveServer(t *testing.T) http.Handler { + t.Helper() + s, _ := store.Open(":memory:") + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i, subj := range []string{"First post", "Second post", "Lunch"} { + s.ArchiveMessage(l.ID, store.Message{ + MessageID: fmt.Sprintf("<%d@x>", i), Subject: subj, Sender: "Alice", + Date: time.Unix(1_700_000_000+int64(i), 0), Body: "body of " + subj, + }) + } + return New(s, token, nil) +} + +type archiveJSON struct { + Total int `json:"total"` + Messages []struct { + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + Subject string `json:"subject"` + Sender string `json:"sender"` + Body string `json:"body"` + } `json:"messages"` +} + +func TestArchivePolicyViaAPI(t *testing.T) { + h := newServer(t) + l := createList(t, h) + if l.ArchivePolicy != "private" { + t.Fatalf("default = %q", l.ArchivePolicy) + } + rec := do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"public"}`, true) + expect(t, rec, 200) + var u listJSON + decode(t, rec, &u) + if u.ArchivePolicy != "public" || u.PostPolicy != "members" { + t.Fatalf("patched: %+v", u) + } + expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"bogus"}`, true), 400) +} + +func TestArchiveList(t *testing.T) { + h := archiveServer(t) + rec := do(t, h, "GET", "/api/v1/lists/1/archive", "", true) + expect(t, rec, 200) + var out archiveJSON + decode(t, rec, &out) + if out.Total != 3 || len(out.Messages) != 3 || out.Messages[0].Subject != "Lunch" { + t.Fatalf("archive: %+v", out) + } + if out.Messages[0].Body != "" { + t.Fatal("listings must not include bodies") + } + + rec = do(t, h, "GET", "/api/v1/lists/1/archive?limit=2&offset=2", "", true) + decode(t, rec, &out) + if out.Total != 3 || len(out.Messages) != 1 || out.Messages[0].Subject != "First post" { + t.Fatalf("page: %+v", out) + } + + rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=post", "", true) + decode(t, rec, &out) + if out.Total != 2 || len(out.Messages) != 2 { + t.Fatalf("search: %+v", out) + } + rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=nothingmatches", "", true) + decode(t, rec, &out) + if out.Total != 0 || out.Messages == nil || len(out.Messages) != 0 { + t.Fatalf("empty search must be an empty array: %+v", out) + } + expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=abc", "", true), 400) + expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?offset=-1", "", true), 400) + expect(t, do(t, h, "GET", "/api/v1/lists/99/archive", "", true), 404) + expect(t, do(t, h, "GET", "/api/v1/lists/1/archive", "", false), 401) +} + +func TestArchiveLimitIsCapped(t *testing.T) { + s, _ := store.Open(":memory:") + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i := 0; i < maxPageSize+5; i++ { + s.ArchiveMessage(l.ID, store.Message{MessageID: fmt.Sprintf("<%d@x>", i), Subject: "s", Date: time.Unix(int64(i), 0)}) + } + h := New(s, token, nil) + rec := do(t, h, "GET", "/api/v1/lists/1/archive?limit=100000", "", true) + expect(t, rec, 200) + var out archiveJSON + decode(t, rec, &out) + if len(out.Messages) != maxPageSize || out.Total != maxPageSize+5 { + t.Fatalf("got %d messages, total %d", len(out.Messages), out.Total) + } + rec = do(t, h, "GET", "/api/v1/lists/1/archive?limit=0", "", true) + decode(t, rec, &out) + if len(out.Messages) != 1 { + t.Fatalf("limit=0 must still return one message, got %d", len(out.Messages)) + } +} + +func TestArchiveGetAndDelete(t *testing.T) { + h := archiveServer(t) + rec := do(t, h, "GET", "/api/v1/archive/1", "", true) + expect(t, rec, 200) + var m struct { + Subject string `json:"subject"` + Body string `json:"body"` + } + decode(t, rec, &m) + if m.Subject != "First post" || m.Body != "body of First post" { + t.Fatalf("message: %+v", m) + } + expect(t, do(t, h, "GET", "/api/v1/archive/99", "", true), 404) + expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 204) + expect(t, do(t, h, "GET", "/api/v1/archive/1", "", true), 404) + expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 404) + expect(t, do(t, h, "DELETE", "/api/v1/archive/2", "", false), 401) +} diff --git a/internal/app/app.go b/internal/app/app.go index 07478ce..55fc292 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -20,6 +20,7 @@ import ( "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/router" "codeberg.org/squidd/parsimony/internal/store" + "codeberg.org/squidd/parsimony/internal/web" ) // App is a running daemon. @@ -28,11 +29,13 @@ type App struct { // APIAddr is empty when the API is disabled. LMTPAddr string APIAddr string + WebAddr string // empty when the public web is disabled store *store.Store lmtp *smtp.Server lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it web *http.Server + pub *http.Server wg sync.WaitGroup once sync.Once } @@ -83,6 +86,21 @@ func Start(cfg config.Config, apiToken string) (*App, error) { a.web = &http.Server{Handler: api.New(st, apiToken, releaser{rt, relay}), ReadHeaderTimeout: 10 * time.Second} } + var pubLn net.Listener + if cfg.Web.Listen != "" { + pubLn, err = net.Listen("tcp", cfg.Web.Listen) + if err != nil { + for _, ln := range a.lns { + ln.Close() + } + st.Close() + return nil, fmt.Errorf("listen web: %w", err) + } + a.WebAddr = pubLn.Addr().String() + a.lns = append(a.lns, pubLn) + a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second} + } + a.wg.Add(1) go func() { defer a.wg.Done() @@ -99,6 +117,15 @@ func Start(cfg config.Config, apiToken string) (*App, error) { } }() } + if a.pub != nil { + a.wg.Add(1) + go func() { + defer a.wg.Done() + if err := a.pub.Serve(pubLn); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + log.Printf("web: %v", err) + } + }() + } return a, nil } @@ -112,6 +139,9 @@ func (a *App) Close() { if a.web != nil { a.web.Close() } + if a.pub != nil { + a.pub.Close() + } a.wg.Wait() a.store.Close() }) diff --git a/internal/app/app_test.go b/internal/app/app_test.go index a984a65..020331b 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -58,6 +58,12 @@ func (s *sink) count() int { return len(s.msgs) } +func (s *sink) all() []sunk { + s.mu.Lock() + defer s.mu.Unlock() + return append([]sunk(nil), s.msgs...) +} + func startSink(t *testing.T) (string, *sink) { t.Helper() sk := &sink{} @@ -79,6 +85,7 @@ func testConfig(t *testing.T, relay string) config.Config { c.LMTP.Addr = "127.0.0.1:0" c.Relay.Addr = relay c.API.Listen = "127.0.0.1:0" + c.Web.Listen = "127.0.0.1:0" return c } @@ -96,6 +103,18 @@ func call(t *testing.T, a *App, method, path, body, token string) int { return resp.StatusCode } +func fetch(t *testing.T, method, url string) (int, string) { + t.Helper() + req, _ := http.NewRequest(method, url, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(b) +} + func lmtpSend(t *testing.T, addr, from, rcpt, msg string) error { t.Helper() conn, err := net.Dial("tcp", addr) @@ -210,3 +229,123 @@ func TestCloseIsIdempotentAndStopsListeners(t *testing.T) { t.Fatal("LMTP still accepting after Close") } } + +func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) { + relayAddr, relay := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.BaseURL = "http://placeholder.invalid" + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + if a.WebAddr == "" { + t.Fatal("web must be listening") + } + + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") + for _, m := range []string{"alice@example.com", "bob@example.com"} { + call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok") + } + web := "http://" + a.WebAddr + + // Private by default: stored but not on the web. + msg := "From: Alice \r\nSubject: Quarterly plan\r\nMessage-ID: \r\n\r\nmail me at alice@example.com\r\n" + if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { + t.Fatal(err) + } + if code, _ := fetch(t, "GET", web+"/archive/dev@example.org"); code != 404 { + t.Fatalf("private archive must be 404, got %d", code) + } + if code := call(t, a, "GET", "/api/v1/lists/1/archive?q=quarterly", "", "tok"); code != 200 { + t.Fatalf("api archive = %d", code) + } + + // Make it public. + if code := call(t, a, "PATCH", "/api/v1/lists/1", `{"archive_policy":"public"}`, "tok"); code != 200 { + t.Fatalf("patch = %d", code) + } + code, body := fetch(t, "GET", web+"/archive/dev@example.org?q=quarterly") + if code != 200 || !strings.Contains(body, "Quarterly plan") { + t.Fatalf("public archive: %d %.200s", code, body) + } + code, body = fetch(t, "GET", web+"/archive/dev@example.org/1") + if code != 200 || !strings.Contains(body, "alice@…") || strings.Contains(body, "alice@example.com") { + t.Fatalf("message page must hide addresses: %d %.300s", code, body) + } + + // The relayed copy carries a working one-click link for that recipient. + var link string + for _, m := range relay.all() { + if len(m.to) == 1 && m.to[0] == "bob@example.com" { + i := strings.Index(m.data, "List-Unsubscribe: ")] + } + } + if link == "" { + t.Fatal("bob got no copy") + } + if code, _ := fetch(t, "GET", web+link); code != 200 { + t.Fatalf("confirm page = %d", code) + } + if code, _ := fetch(t, "POST", web+link); code != 200 { + t.Fatalf("one-click = %d", code) + } + before := relay.count() + if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", + "From: Alice \r\nSubject: Second\r\nMessage-ID: \r\n\r\nhi\r\n"); err != nil { + t.Fatal(err) + } + if got := relay.count() - before; got != 1 { + t.Fatalf("after bob left only alice should get a copy, got %d", got) + } +} + +func TestWebDisabledWhenListenIsEmpty(t *testing.T) { + relayAddr, _ := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.Web.Listen = "" + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + if a.WebAddr != "" { + t.Fatalf("web must be off, listening on %s", a.WebAddr) + } +} + +func TestCloseDropsOpenConnections(t *testing.T) { + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + conns := map[string]net.Conn{} + for name, addr := range map[string]string{"web": a.WebAddr, "api": a.APIAddr, "lmtp": a.LMTPAddr} { + c, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + conns[name] = c + } + time.Sleep(100 * time.Millisecond) // let the servers accept + a.Close() + for name, c := range conns { + c.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 512) + for { + if _, err := c.Read(buf); err != nil { + if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Errorf("%s: connection still open after Close", name) + } + break + } + } + } +} diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..afc76d9 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,179 @@ +// Package archive extracts what the archive needs from a raw email message. +package archive + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "html" + "io" + "regexp" + "strings" + "time" + "unicode/utf8" + + "github.com/emersion/go-message" + "github.com/emersion/go-message/mail" +) + +// Register charset decoders (iso-8859-*, windows-125x, ...). +import _ "github.com/emersion/go-message/charset" + +// MaxBody is the largest body stored for one message, in bytes. +const MaxBody = 1 << 20 + +// Entry is the archivable content of a message. +type Entry struct { + MessageID string + InReplyTo string + Subject string + Sender string // display name only, never the address domain + Date time.Time + Body string // plain text, UTF-8 +} + +var ( + msgIDRe = regexp.MustCompile(`<[^<>\s]+>`) + scriptRe = regexp.MustCompile(`(?is)]*>.*?`) + styleRe = regexp.MustCompile(`(?is)]*>.*?`) + breakRe = regexp.MustCompile(`(?i)|`) + tagRe = regexp.MustCompile(`<[^>]*>`) + blankRe = regexp.MustCompile(`\n{3,}`) + trailingRe = regexp.MustCompile(`[ \t]+\n`) + emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`) +) + +// Parse reads an RFC 5322 message. now is used when the Date header is missing +// or invalid. +func Parse(raw []byte, now time.Time) (Entry, error) { + ent, err := message.Read(bytes.NewReader(raw)) + if err != nil && !message.IsUnknownCharset(err) { + return Entry{}, fmt.Errorf("parse message: %w", err) + } + if ent == nil || ent.Header.Len() == 0 { + return Entry{}, errors.New("parse message: no headers") + } + h := mail.Header{Header: ent.Header} + + e := Entry{Date: now} + if d, err := h.Date(); err == nil && !d.IsZero() { + e.Date = d + } + if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" { + e.Subject = "(no subject)" + } + e.Subject = clean(strings.Join(strings.Fields(e.Subject), " ")) + e.Sender = senderName(h) + + if id := msgIDRe.FindString(ent.Header.Get("Message-Id")); id != "" { + e.MessageID = id + } else { + sum := sha256.Sum256(raw) + e.MessageID = "<" + hex.EncodeToString(sum[:12]) + "@parsimony.invalid>" + } + if id := msgIDRe.FindString(ent.Header.Get("In-Reply-To")); id != "" { + e.InReplyTo = id + } else if refs := msgIDRe.FindAllString(ent.Header.Get("References"), -1); len(refs) > 0 { + e.InReplyTo = refs[len(refs)-1] + } + + plain, htmlBody := bodies(ent) + if strings.TrimSpace(plain) == "" && htmlBody != "" { + plain = stripHTML(htmlBody) + } + e.Body = clean(plain) + return e, nil +} + +func senderName(h mail.Header) string { + list, err := h.AddressList("From") + if err != nil || len(list) == 0 { + return "" + } + if n := strings.TrimSpace(list[0].Name); n != "" { + return clean(n) + } + local, _, _ := strings.Cut(list[0].Address, "@") + return clean(local) +} + +// bodies returns the first plain text and the first HTML body found. +func bodies(e *message.Entity) (plain, htmlBody string) { + ct, _, _ := e.Header.ContentType() + if ct == "" { + ct = "text/plain" + } + if strings.HasPrefix(ct, "multipart/") { + mr := e.MultipartReader() + if mr == nil { + return "", "" + } + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil && !message.IsUnknownCharset(err) { + break + } + if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" { + continue + } + p, h := bodies(part) + if plain == "" { + plain = p + } + if htmlBody == "" { + htmlBody = h + } + } + return plain, htmlBody + } + switch ct { + case "text/plain": + plain = readLimited(e.Body) + case "text/html": + htmlBody = readLimited(e.Body) + } + return plain, htmlBody +} + +func readLimited(r io.Reader) string { + b, _ := io.ReadAll(io.LimitReader(r, MaxBody+4)) + return string(b) +} + +func stripHTML(s string) string { + s = scriptRe.ReplaceAllString(s, "") + s = styleRe.ReplaceAllString(s, "") + s = breakRe.ReplaceAllString(s, "\n") + s = tagRe.ReplaceAllString(s, "") + return html.UnescapeString(s) +} + +// clean makes text safe to store: valid UTF-8, no NUL, LF line ends, capped. +func clean(s string) string { + s = strings.ToValidUTF8(s, "\uFFFD") + s = strings.ReplaceAll(s, "\x00", "") + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + if len(s) > MaxBody { + s = s[:MaxBody] + for !utf8.ValidString(s) { // do not cut a character in half + s = s[:len(s)-1] + } + } + if strings.Contains(s, "\n\n\n") || strings.Contains(s, " \n") { + s = trailingRe.ReplaceAllString(s, "\n") + s = blankRe.ReplaceAllString(s, "\n\n") + } + return s +} + +// Obfuscate hides the domain of email addresses so a public archive cannot be +// harvested for them: alice@example.com becomes alice@…. +func Obfuscate(s string) string { + return emailRe.ReplaceAllString(s, "${1}@…") +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 0000000..fa96e29 --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,214 @@ +package archive + +import ( + "strings" + "testing" + "time" + "unicode/utf8" +) + +var now = time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC) + +func parse(t *testing.T, raw string) Entry { + t.Helper() + e, err := Parse([]byte(raw), now) + if err != nil { + t.Fatalf("parse: %v", err) + } + return e +} + +func TestPlainMessage(t *testing.T) { + e := parse(t, "From: Alice Smith \r\nSubject: Hello\r\nMessage-ID: \r\n"+ + "Date: Mon, 02 Jan 2006 15:04:05 +0000\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nline one\r\nline two\r\n") + if e.Subject != "Hello" || e.Sender != "Alice Smith" || e.MessageID != "" { + t.Fatalf("entry: %+v", e) + } + if e.Body != "line one\nline two\n" { + t.Fatalf("body = %q", e.Body) + } + if !e.Date.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)) { + t.Fatalf("date = %v", e.Date) + } +} + +func TestSenderWithoutNameNeverExposesTheDomain(t *testing.T) { + e := parse(t, "From: bob@secret.example.com\r\nSubject: x\r\n\r\nhi\r\n") + if e.Sender != "bob" { + t.Fatalf("sender = %q", e.Sender) + } +} + +func TestEncodedSubjectAndSender(t *testing.T) { + e := parse(t, "From: =?UTF-8?B?w4lsw6g=?= \r\nSubject: =?ISO-8859-1?Q?Caf=E9_ol=E9?=\r\n\r\nx\r\n") + if e.Subject != "Café olé" || e.Sender != "Él\u00e8" { + t.Fatalf("entry: %q %q", e.Subject, e.Sender) + } +} + +func TestBase64UTF8Body(t *testing.T) { + e := parse(t, "Subject: b\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: base64\r\n\r\n"+ + "Y2lhbyBtb25kbyDimIM=\r\n") + if strings.TrimSpace(e.Body) != "ciao mondo ☃" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestQuotedPrintableLatin1Body(t *testing.T) { + e := parse(t, "Subject: q\r\nContent-Type: text/plain; charset=iso-8859-1\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n"+ + "caff=E8 l=E0\r\n") + if strings.TrimSpace(e.Body) != "caffè là" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestMultipartAlternativePrefersPlain(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/alternative; boundary=BB\r\n\r\n"+ + "--BB\r\nContent-Type: text/html\r\n\r\n

html version

\r\n"+ + "--BB\r\nContent-Type: text/plain\r\n\r\nplain version\r\n--BB--\r\n") + if strings.TrimSpace(e.Body) != "plain version" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestMultipartMixedWithAttachmentAndNesting(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=OUT\r\n\r\n"+ + "--OUT\r\nContent-Type: multipart/alternative; boundary=IN\r\n\r\n"+ + "--IN\r\nContent-Type: text/plain\r\n\r\nreal text\r\n--IN--\r\n"+ + "--OUT\r\nContent-Type: application/pdf\r\nContent-Disposition: attachment; filename=a.pdf\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--OUT--\r\n") + if strings.TrimSpace(e.Body) != "real text" { + t.Fatalf("body = %q", e.Body) + } + if strings.Contains(e.Body, "JVBER") { + t.Fatal("attachments must not leak into the body") + } +} + +func TestHTMLOnlyIsStripped(t *testing.T) { + e := parse(t, "Subject: h\r\nContent-Type: text/html\r\n\r\n"+ + "

Hello & welcome

second

\r\n") + if strings.ContainsAny(e.Body, "<>") || strings.Contains(e.Body, "alert") || strings.Contains(e.Body, "color") { + t.Fatalf("html not stripped: %q", e.Body) + } + if !strings.Contains(e.Body, "Hello & welcome") || !strings.Contains(e.Body, "second") { + t.Fatalf("text lost: %q", e.Body) + } +} + +func TestUnknownCharsetStillArchives(t *testing.T) { + e := parse(t, "Subject: u\r\nContent-Type: text/plain; charset=x-made-up\r\n\r\nstill here\r\n") + if !strings.Contains(e.Body, "still here") { + t.Fatalf("body = %q", e.Body) + } +} + +func TestMissingMessageIDIsSynthesizedAndStable(t *testing.T) { + raw := "From: a@x.com\r\nSubject: no id\r\n\r\nbody\r\n" + a, b := parse(t, raw), parse(t, raw) + if a.MessageID == "" || !strings.HasPrefix(a.MessageID, "<") || a.MessageID != b.MessageID { + t.Fatalf("ids: %q %q", a.MessageID, b.MessageID) + } + if c := parse(t, raw+"x"); c.MessageID == a.MessageID { + t.Fatal("different messages need different ids") + } +} + +func TestInReplyToAndReferences(t *testing.T) { + e := parse(t, "Subject: r\r\nIn-Reply-To: (comment)\r\n\r\nx\r\n") + if e.InReplyTo != "" { + t.Fatalf("in-reply-to = %q", e.InReplyTo) + } + e = parse(t, "Subject: r\r\nReferences: \r\n\r\nx\r\n") + if e.InReplyTo != "" { + t.Fatalf("references fallback = %q", e.InReplyTo) + } + if e = parse(t, "Subject: r\r\n\r\nx\r\n"); e.InReplyTo != "" { + t.Fatalf("none expected, got %q", e.InReplyTo) + } +} + +func TestBadOrMissingDateFallsBackToNow(t *testing.T) { + for _, h := range []string{"", "Date: not a date\r\n"} { + if e := parse(t, h+"Subject: d\r\n\r\nx\r\n"); !e.Date.Equal(now) { + t.Fatalf("%q: date = %v", h, e.Date) + } + } +} + +func TestMissingSubject(t *testing.T) { + if e := parse(t, "From: a@x.com\r\n\r\nx\r\n"); e.Subject != "(no subject)" { + t.Fatalf("subject = %q", e.Subject) + } +} + +func TestBodyIsCappedAndCleaned(t *testing.T) { + big := strings.Repeat("a", MaxBody+5000) + e := parse(t, "Subject: big\r\n\r\n"+big+"\r\n") + if len(e.Body) > MaxBody { + t.Fatalf("body len %d exceeds cap", len(e.Body)) + } + e = parse(t, "Subject: nul\r\n\r\nab\x00cd\r\n") + if strings.Contains(e.Body, "\x00") { + t.Fatal("NUL must be removed") + } +} + +func TestInvalidUTF8IsRepaired(t *testing.T) { + e := parse(t, "Subject: bad\r\nContent-Type: text/plain\r\n\r\nok \xff\xfe end\r\n") + if !strings.Contains(e.Body, "ok") || !strings.Contains(e.Body, "end") { + t.Fatalf("body = %q", e.Body) + } + if strings.ToValidUTF8(e.Body, "") != e.Body { + t.Fatal("body must be valid UTF-8") + } +} + +func TestEmptyInputIsAnError(t *testing.T) { + if _, err := Parse(nil, now); err == nil { + t.Fatal("want error for an unparseable message") + } +} + +func TestObfuscate(t *testing.T) { + cases := map[string]string{ + "mail alice@example.com now": "mail alice@… now", + "a.b+c@sub.example.org, x@y.io": "a.b+c@…, x@…", + "no address here": "no address here", + "": "", + "twitter @handle only": "twitter @handle only", + } + for in, want := range cases { + if got := Obfuscate(in); got != want { + t.Errorf("Obfuscate(%q) = %q, want %q", in, got, want) + } + } +} + +func TestTextAttachmentIsNotTheBody(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=B\r\n\r\n"+ + "--B\r\nContent-Type: text/plain\r\nContent-Disposition: attachment; filename=log.txt\r\n\r\nattached log\r\n"+ + "--B\r\nContent-Type: text/plain\r\n\r\nthe real message\r\n--B--\r\n") + if strings.TrimSpace(e.Body) != "the real message" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestFirstTextPartWins(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=B\r\n\r\n"+ + "--B\r\nContent-Type: text/plain\r\n\r\nmessage\r\n"+ + "--B\r\nContent-Type: text/plain\r\n\r\n-- \r\nsignature footer\r\n--B--\r\n") + if strings.TrimSpace(e.Body) != "message" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestCapNeverSplitsAMultibyteCharacter(t *testing.T) { + // 3-byte characters: MaxBody is not a multiple of 3, so a blind cut lands mid-character. + e := parse(t, "Subject: big\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+strings.Repeat("€", MaxBody/3+10)+"\r\n") + if !utf8.ValidString(e.Body) || strings.ContainsRune(e.Body, utf8.RuneError) { + t.Fatal("body was cut in the middle of a character") + } + if len(e.Body) > MaxBody || len(e.Body) < MaxBody-3 { + t.Fatalf("body len %d", len(e.Body)) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 83027b5..dd33a8b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,6 +20,9 @@ type Config struct { API struct { Listen string `yaml:"listen"` // HTTP listen address; the bearer token comes from PARSIMONY_API_TOKEN } `yaml:"api"` + Web struct { + Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them + } `yaml:"web"` Relay struct { Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay } `yaml:"relay"` @@ -33,6 +36,7 @@ func Load(path string) (Config, error) { c.LMTP.Addr = "127.0.0.1:2424" c.Relay.Addr = "127.0.0.1:25" c.API.Listen = "127.0.0.1:8080" + c.Web.Listen = "127.0.0.1:8081" b, err := os.ReadFile(path) if err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 99d0729..0e39f69 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -20,17 +20,17 @@ func TestLoadDefaults(t *testing.T) { if err != nil { t.Fatal(err) } - if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.API.Listen != "127.0.0.1:8080" { + if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" { t.Fatalf("defaults: %+v", c) } } func TestLoadValues(t *testing.T) { - c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\napi:\n listen: 0.0.0.0:9000\n")) + c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n")) if err != nil { t.Fatal(err) } - if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.API.Listen != "0.0.0.0:9000" { + if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" { t.Fatalf("values: %+v", c) } } @@ -46,3 +46,13 @@ func TestLoadUnknownField(t *testing.T) { t.Fatal("unknown fields must be rejected") } } + +func TestWebCanBeDisabled(t *testing.T) { + c, err := Load(write(t, "web:\n listen: \"\"\n")) + if err != nil { + t.Fatal(err) + } + if c.Web.Listen != "" { + t.Fatalf("an empty listen address disables the web: %q", c.Web.Listen) + } +} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 051c042..4535268 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -180,7 +180,7 @@ func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) { t.Cleanup(func() { s.Close() }) l, _ := s.CreateList("dev", "example.org", "Dev") s.AddMember(l.ID, "alice@example.com", "Alice") - s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen) + s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) relay := &outbound.Fake{} srv := NewServer(router.New(s), relay) ln, _ := net.Listen("tcp", "127.0.0.1:0") diff --git a/internal/router/commands.go b/internal/router/commands.go index 97d852d..5e3b131 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -28,19 +28,21 @@ func (r *Router) secret() ([]byte, error) { return r.secretVal, r.secretErr } -func (r *Router) mac(memberID int64) string { +// mac signs a member id for one purpose ("bounce" or "unsub"), so a token +// made for one use cannot be replayed for another. +func (r *Router) mac(purpose string, memberID int64) string { key, err := r.secret() if err != nil { return "" } h := hmac.New(sha256.New, key) - fmt.Fprintf(h, "bounce:%d", memberID) + fmt.Fprintf(h, "%s:%d", purpose, memberID) return hex.EncodeToString(h.Sum(nil))[:12] } // bounceAddress returns the VERP envelope sender for a member, or "" on error. func (r *Router) bounceAddress(l store.List, memberID int64) string { - mac := r.mac(memberID) + mac := r.mac("bounce", memberID) if mac == "" { return "" } @@ -156,7 +158,7 @@ func (r *Router) bounce(l store.List, tag string, data []byte) error { if !ok || err != nil { return ErrBadToken } - want := r.mac(id) + want := r.mac("bounce", id) if want == "" || !hmac.Equal([]byte(mac), []byte(want)) { return ErrBadToken } @@ -172,3 +174,57 @@ func (r *Router) bounce(l store.List, tag string, data []byte) error { } return err } + +// UnsubscribeToken returns the signed token used in one-click unsubscribe links +// for a member, or "" if the secret is unavailable. +func (r *Router) UnsubscribeToken(memberID int64) string { + mac := r.mac("unsub", memberID) + if mac == "" { + return "" + } + return fmt.Sprintf("%d-%s", memberID, mac) +} + +// parseUnsubscribe verifies a token and returns the member id. +func (r *Router) parseUnsubscribe(token string) (int64, error) { + idStr, mac, ok := strings.Cut(token, "-") + id, err := strconv.ParseInt(idStr, 10, 64) + if !ok || err != nil || id <= 0 { + return 0, ErrBadToken + } + want := r.mac("unsub", id) + if want == "" || !hmac.Equal([]byte(mac), []byte(want)) { + return 0, ErrBadToken + } + return id, nil +} + +// CheckUnsubscribe validates a token and returns the list it belongs to, +// without changing anything. ErrGone means the member is already unsubscribed. +func (r *Router) CheckUnsubscribe(token string) (store.List, error) { + id, err := r.parseUnsubscribe(token) + if err != nil { + return store.List{}, err + } + m, err := r.store.MemberByID(id) + if errors.Is(err, store.ErrNotFound) { + return store.List{}, ErrGone + } + if err != nil { + return store.List{}, err + } + return r.store.GetList(m.ListID) +} + +// Unsubscribe removes the member behind a valid token. Repeating it succeeds, +// because mail clients may send the one-click request more than once. +func (r *Router) Unsubscribe(token string) error { + id, err := r.parseUnsubscribe(token) + if err != nil { + return err + } + if err := r.store.DeleteMember(id); err != nil && !errors.Is(err, store.ErrNotFound) { + return err + } + return nil +} diff --git a/internal/router/headers.go b/internal/router/headers.go index 1763a75..8847dea 100644 --- a/internal/router/headers.go +++ b/internal/router/headers.go @@ -6,16 +6,15 @@ import ( "fmt" "io" "net/mail" - "strings" "github.com/emersion/go-message/textproto" "codeberg.org/squidd/parsimony/internal/store" ) -// rewrite adds list headers and rewrites From so DMARC alignment holds. +// rewrite adds the list headers shared by every recipient and rewrites From so DMARC alignment holds. // The body is copied byte for byte. -func rewrite(l store.List, baseURL string, data []byte) ([]byte, error) { +func rewrite(l store.List, data []byte) ([]byte, error) { br := bufio.NewReader(bytes.NewReader(data)) h, err := textproto.ReadHeader(br) if err != nil { @@ -30,12 +29,7 @@ func rewrite(l store.List, baseURL string, data []byte) ([]byte, error) { id := l.Name + "." + l.Domain h.Set("List-Id", fmt.Sprintf("%s <%s>", l.Title, id)) h.Set("List-Post", "") - unsub := "" - if baseURL != "" { - unsub = "<" + strings.TrimRight(baseURL, "/") + "/unsubscribe/" + id + ">, " + unsub - h.Set("List-Unsubscribe-Post", "List-Unsubscribe=One-Click") - } - h.Set("List-Unsubscribe", unsub) + // List-Unsubscribe is per recipient: Router.Send adds it. h.Set("Precedence", "list") if orig := h.Get("From"); orig != "" { diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index 48caf8f..c1b0e20 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -85,7 +85,7 @@ func TestDisabledMembersAreSkipped(t *testing.T) { func TestModeratedListHoldsPosts(t *testing.T) { e := newEnv(t) - e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen) + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) for _, sender := range []string{"alice@example.com", "stranger@x.com"} { _, err := e.r.Route(sender, "dev@example.org", []byte(sample)) if !errors.Is(err, ErrHeld) { @@ -103,7 +103,7 @@ func TestModeratedListHoldsPosts(t *testing.T) { func TestReleaseHeld(t *testing.T) { e := newEnv(t) - e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen) + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) e.r.Route("stranger@x.com", "dev@example.org", []byte(sample)) held, _ := e.s.ListHeld(e.list.ID) @@ -182,7 +182,7 @@ func TestJoinRateLimitedAndSilentForMembers(t *testing.T) { func TestJoinClosed(t *testing.T) { e := newEnv(t) - e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) if !errors.Is(err, ErrJoinClosed) { t.Fatalf("want ErrJoinClosed, got %v", err) @@ -196,7 +196,7 @@ func TestConfirmAfterListClosed(t *testing.T) { e := newEnv(t) e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) tok := token(t, lastMail(t, e.out)) - e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) if !errors.Is(err, ErrJoinClosed) { t.Fatalf("want ErrJoinClosed, got %v", err) diff --git a/internal/router/m3_test.go b/internal/router/m3_test.go new file mode 100644 index 0000000..6769095 --- /dev/null +++ b/internal/router/m3_test.go @@ -0,0 +1,214 @@ +package router + +import ( + "errors" + "fmt" + "regexp" + "strings" + "testing" + + "codeberg.org/squidd/parsimony/internal/store" +) + +const post = "From: Alice Smith \r\nTo: dev@example.org\r\nSubject: Hello list\r\n" + + "Message-ID: \r\nContent-Type: text/plain\r\n\r\nthe body\r\n" + +func TestAcceptedPostIsArchivedFromTheOriginal(t *testing.T) { + e := newEnv(t) + if _, err := e.r.Route("alice@example.com", "dev@example.org", []byte(post)); err != nil { + t.Fatal(err) + } + ms, total, _ := e.s.ListMessages(e.list.ID, 10, 0) + if total != 1 { + t.Fatalf("archived = %d", total) + } + m, _ := e.s.GetMessage(ms[0].ID) + if m.Subject != "Hello list" || m.Body != "the body\n" || m.MessageID != "" { + t.Fatalf("message: %+v", m) + } + if m.Sender != "Alice Smith" { + t.Fatalf("sender must be the original author, not the list rewrite: %q", m.Sender) + } +} + +func TestArchivePolicyOffStoresNothing(t *testing.T) { + e := newEnv(t) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff) + e.r.Route("alice@example.com", "dev@example.org", []byte(post)) + if _, total, _ := e.s.ListMessages(e.list.ID, 10, 0); total != 0 { + t.Fatalf("archived = %d", total) + } +} + +func TestRetryDoesNotDuplicateTheArchive(t *testing.T) { + e := newEnv(t) + e.r.Route("alice@example.com", "dev@example.org", []byte(post)) + e.r.Route("alice@example.com", "dev@example.org", []byte(post)) + if _, total, _ := e.s.ListMessages(e.list.ID, 10, 0); total != 1 { + t.Fatalf("archived = %d", total) + } +} + +func TestRejectedAndHeldPostsAreNotArchived(t *testing.T) { + e := newEnv(t) + e.r.Route("mallory@evil.com", "dev@example.org", []byte(post)) // rejected + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) + e.r.Route("alice@example.com", "dev@example.org", []byte(post)) // held + if _, total, _ := e.s.ListMessages(e.list.ID, 10, 0); total != 0 { + t.Fatalf("archived = %d", total) + } + held, _ := e.s.ListHeld(e.list.ID) + if err := e.r.Release(e.ctx, e.out, held[0].ID); err != nil { + t.Fatal(err) + } + if _, total, _ := e.s.ListMessages(e.list.ID, 10, 0); total != 1 { + t.Fatalf("released post must be archived, got %d", total) + } +} + +func TestUnparseablePostIsStillDelivered(t *testing.T) { + e := newEnv(t) + res, err := e.r.Route("alice@example.com", "dev@example.org", []byte("\r\nno headers at all\r\n")) + if err != nil { + t.Fatalf("archive failures must not block delivery: %v", err) + } + if len(res.Recipients) != 2 { + t.Fatalf("recipients: %+v", res.Recipients) + } +} + +var unsubURL = regexp.MustCompile(``) + +func sendPost(t *testing.T, e *env) { + t.Helper() + res, err := e.r.Route("alice@example.com", "dev@example.org", []byte(post)) + if err != nil { + t.Fatal(err) + } + if err := e.r.Send(e.ctx, e.out, res); err != nil { + t.Fatal(err) + } +} + +func TestEachRecipientGetsItsOwnUnsubscribeLink(t *testing.T) { + e := newEnv(t) + e.r.BaseURL = "https://lists.example.org" + sendPost(t, e) + tokens := map[string]bool{} + for _, m := range e.out.Sent { + h := header(t, m.Data) + lu := h.Get("List-Unsubscribe") + sub := unsubURL.FindStringSubmatch(lu) + if sub == nil || !strings.Contains(lu, "") { + t.Fatalf("List-Unsubscribe = %q", lu) + } + tokens[sub[1]] = true + if h.Get("List-Unsubscribe-Post") != "List-Unsubscribe=One-Click" { + t.Errorf("List-Unsubscribe-Post = %q", h.Get("List-Unsubscribe-Post")) + } + if n := len(h["List-Unsubscribe"]); n != 1 { + t.Errorf("want one List-Unsubscribe header, got %d", n) + } + if h.Get("List-Id") == "" || !strings.HasSuffix(string(m.Data), "\r\n\r\nthe body\r\n") { + t.Errorf("rest of the message was damaged: %q", m.Data) + } + } + if len(tokens) != 2 { + t.Fatalf("each recipient needs a distinct link, got %v", tokens) + } +} + +func TestNoOneClickWithoutBaseURL(t *testing.T) { + e := newEnv(t) + sendPost(t, e) + h := header(t, e.out.Sent[0].Data) + if h.Get("List-Unsubscribe-Post") != "" { + t.Error("one-click needs a base URL") + } + if got := h.Get("List-Unsubscribe"); got != "" { + t.Errorf("List-Unsubscribe = %q", got) + } +} + +func TestSpoofedUnsubscribeHeadersNeverReachRecipients(t *testing.T) { + e := newEnv(t) + msg := "From: Alice \r\nList-Unsubscribe: \r\nList-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n\r\nx\r\n" + res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(msg)) + e.r.Send(e.ctx, e.out, res) + for _, m := range e.out.Sent { + h := header(t, m.Data) + if strings.Contains(h.Get("List-Unsubscribe"), "evil") || h.Get("List-Unsubscribe-Post") != "" { + t.Fatalf("spoofed header survived: %v", h) + } + } +} + +func tokenFor(t *testing.T, e *env, memberID int64) string { + t.Helper() + e.r.BaseURL = "https://lists.example.org" + return e.r.UnsubscribeToken(memberID) +} + +func TestUnsubscribeToken(t *testing.T) { + e := newEnv(t) + tok := tokenFor(t, e, e.alice) + l, err := e.r.CheckUnsubscribe(tok) + if err != nil || l.ID != e.list.ID { + t.Fatalf("check: %+v %v", l, err) + } + if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); !ok { + t.Fatal("checking must not unsubscribe") + } + if err := e.r.Unsubscribe(tok); err != nil { + t.Fatal(err) + } + if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); ok { + t.Fatal("alice must be gone") + } + if ok, _ := e.s.IsMember(e.list.ID, "bob@example.com"); !ok { + t.Fatal("bob must stay") + } + // Mail clients may repeat the request. + if err := e.r.Unsubscribe(tok); err != nil { + t.Fatalf("repeat must succeed: %v", err) + } + if _, err := e.r.CheckUnsubscribe(tok); !errors.Is(err, ErrGone) { + t.Fatalf("want ErrGone, got %v", err) + } +} + +func TestUnsubscribeRejectsForgedTokens(t *testing.T) { + e := newEnv(t) + for _, tok := range []string{"", "x", "1", "1-", "-1", fmt.Sprintf("%d-000000000000", e.alice), "abc-123456789012", + fmt.Sprintf("%d-%s", e.alice+1, strings.TrimPrefix(tokenFor(t, e, e.alice), fmt.Sprintf("%d-", e.alice)))} { + if err := e.r.Unsubscribe(tok); !errors.Is(err, ErrBadToken) { + t.Errorf("token %q: want ErrBadToken, got %v", tok, err) + } + } + if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); !ok { + t.Fatal("forged tokens must not unsubscribe anyone") + } +} + +func TestBounceAndUnsubscribeTokensAreNotInterchangeable(t *testing.T) { + e := newEnv(t) + bounceMAC := e.r.mac("bounce", e.alice) + unsubMAC := e.r.mac("unsub", e.alice) + if bounceMAC == unsubMAC { + t.Fatal("MACs must be bound to their purpose") + } + if err := e.r.Unsubscribe(fmt.Sprintf("%d-%s", e.alice, bounceMAC)); !errors.Is(err, ErrBadToken) { + t.Fatalf("a bounce MAC must not unsubscribe: %v", err) + } +} + +func TestUnsubscribeRejectsMemberIDZero(t *testing.T) { + e := newEnv(t) + tok := fmt.Sprintf("0-%s", e.r.mac("unsub", 0)) // correctly signed, but not a real member id + if err := e.r.Unsubscribe(tok); !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } + if _, err := e.r.CheckUnsubscribe(tok); !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 0bf7d91..43d5989 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -6,8 +6,11 @@ import ( "errors" "fmt" "log" + "strings" "sync" + "time" + "codeberg.org/squidd/parsimony/internal/archive" "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/store" ) @@ -21,6 +24,8 @@ var ( ErrHeld = errors.New("held for moderation") // ErrBadToken means a confirmation or bounce token is invalid, used or expired. ErrBadToken = errors.New("invalid or expired token") + // ErrGone means the member behind a valid token is already unsubscribed. + ErrGone = errors.New("already unsubscribed") // ErrJoinClosed means the list does not accept self-service subscriptions. ErrJoinClosed = errors.New("list is closed") ) @@ -88,7 +93,8 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) { for i, m := range members { rcpts[i] = Recipient{ID: m.ID, Address: m.Address} } - out, err := rewrite(l, r.BaseURL, data) + r.archive(l, data) + out, err := rewrite(l, data) if err != nil { return Result{}, err } @@ -105,7 +111,8 @@ func (r *Router) Send(ctx context.Context, out outbound.Sender, res Result) erro if from == "" { return errors.New("cannot create bounce address") } - if err := out.Send(ctx, from, []string{rc.Address}, res.Data); err != nil { + data := append(r.unsubscribeHeaders(res.List, rc.ID), res.Data...) + if err := out.Send(ctx, from, []string{rc.Address}, data); err != nil { failed++ log.Printf("deliver %s to %s: %v", res.List.Address(), rc.Address, err) } @@ -116,6 +123,39 @@ func (r *Router) Send(ctx context.Context, out outbound.Sender, res Result) erro return nil } +// unsubscribeHeaders builds the per-recipient List-Unsubscribe headers: a signed +// one-click link (RFC 8058) when BaseURL is set, and the +leave address. +func (r *Router) unsubscribeHeaders(l store.List, memberID int64) []byte { + value := "" + var b strings.Builder + if tok := r.UnsubscribeToken(memberID); r.BaseURL != "" && tok != "" { + value = "<" + strings.TrimRight(r.BaseURL, "/") + "/unsubscribe/" + tok + ">, " + value + b.WriteString("List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n") + } + b.WriteString("List-Unsubscribe: " + value + "\r\n") + return []byte(b.String()) +} + +// archive stores the original message. Failures are logged, never fatal: the +// post must be delivered even if the archive is unavailable. +func (r *Router) archive(l store.List, data []byte) { + if l.ArchivePolicy == store.ArchiveOff { + return + } + en, err := archive.Parse(data, time.Now()) + if err != nil { + log.Printf("archive %s: %v", l.Address(), err) + return + } + _, _, err = r.store.ArchiveMessage(l.ID, store.Message{ + MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject, + Sender: en.Sender, Date: en.Date, Body: en.Body, + }) + if err != nil { + log.Printf("archive %s: %v", l.Address(), err) + } +} + // Release approves a held message and delivers it. func (r *Router) Release(ctx context.Context, out outbound.Sender, heldID int64) error { h, err := r.store.GetHeld(heldID) diff --git a/internal/router/router_test.go b/internal/router/router_test.go index 0acf5e5..a12227e 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -85,13 +85,6 @@ func TestRewriteHeaders(t *testing.T) { if got := h.Get("List-Post"); got != "" { t.Errorf("List-Post = %q", got) } - if got := h.Get("List-Unsubscribe"); !strings.Contains(got, "") || - !strings.Contains(got, "") { - t.Errorf("List-Unsubscribe = %q", got) - } - if got := h.Get("List-Unsubscribe-Post"); got != "List-Unsubscribe=One-Click" { - t.Errorf("List-Unsubscribe-Post = %q", got) - } if got := h.Get("From"); got != `"Alice via Dev" ` { t.Errorf("From = %q", got) } @@ -114,15 +107,3 @@ func TestRewriteKeepsExistingReplyTo(t *testing.T) { t.Errorf("Reply-To = %q", got) } } - -func TestNoOneClickWithoutBaseURL(t *testing.T) { - r := setup(t) - res, _ := r.Route("alice@example.com", "dev@example.org", []byte(sample)) - h := header(t, res.Data) - if h.Get("List-Unsubscribe-Post") != "" { - t.Error("one-click header needs a base URL") - } - if got := h.Get("List-Unsubscribe"); got != "" { - t.Errorf("List-Unsubscribe = %q", got) - } -} diff --git a/internal/store/archive_test.go b/internal/store/archive_test.go new file mode 100644 index 0000000..7fe6b01 --- /dev/null +++ b/internal/store/archive_test.go @@ -0,0 +1,229 @@ +package store + +import ( + "errors" + "strings" + "testing" + "time" +) + +func msg(id, subject, body string) Message { + return Message{MessageID: id, Subject: subject, Sender: "Alice", Date: time.Unix(1_700_000_000, 0), Body: body} +} + +func TestArchivePolicyDefaultsAndUpdate(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if l.ArchivePolicy != ArchivePrivate { + t.Fatalf("default = %q", l.ArchivePolicy) + } + if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, ArchivePublic); err != nil { + t.Fatal(err) + } + got, _ := s.GetList(l.ID) + if got.ArchivePolicy != ArchivePublic { + t.Fatalf("policy = %q", got.ArchivePolicy) + } + if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, "bogus"); err == nil { + t.Fatal("invalid archive policy must be rejected") + } +} + +func TestArchiveAndFetch(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + m := msg("<1@x>", "Hello", "first body") + id, created, err := s.ArchiveMessage(l.ID, m) + if err != nil || !created || id == 0 { + t.Fatalf("archive: %d %v %v", id, created, err) + } + got, err := s.GetMessage(id) + if err != nil { + t.Fatal(err) + } + if got.Subject != "Hello" || got.Body != "first body" || got.ListID != l.ID || got.ThreadID != id || got.Size != len("first body") { + t.Fatalf("got %+v", got) + } + if !got.Date.Equal(m.Date) { + t.Fatalf("date = %v", got.Date) + } + if _, err := s.GetMessage(999); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestArchiveDeduplicatesByMessageID(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + id1, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a")) + id2, created, err := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a")) + if err != nil || created || id2 != id1 { + t.Fatalf("retry must not duplicate: %d %v %v", id2, created, err) + } + other, _ := s.CreateList("ops", "example.org", "Ops") + if _, created, _ := s.ArchiveMessage(other.ID, msg("<1@x>", "Hello", "a")); !created { + t.Fatal("same Message-ID in another list is a different message") + } +} + +func TestThreading(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + root, _, _ := s.ArchiveMessage(l.ID, msg("", "Topic", "r")) + reply := msg("", "Re: Topic", "r1") + reply.InReplyTo = "" + r1, _, _ := s.ArchiveMessage(l.ID, reply) + nested := msg("", "Re: Topic", "r2") + nested.InReplyTo = "" + r2, _, _ := s.ArchiveMessage(l.ID, nested) + orphan := msg("", "Other", "o") + orphan.InReplyTo = "" + o, _, _ := s.ArchiveMessage(l.ID, orphan) + + for _, id := range []int64{r1, r2} { + m, _ := s.GetMessage(id) + if m.ThreadID != root { + t.Fatalf("message %d thread = %d, want %d", id, m.ThreadID, root) + } + } + if m, _ := s.GetMessage(o); m.ThreadID != o { + t.Fatalf("orphan must start its own thread, got %d", m.ThreadID) + } + th, err := s.ThreadMessages(l.ID, root) + if err != nil || len(th) != 3 { + t.Fatalf("thread: %d %v", len(th), err) + } +} + +func TestThreadingIgnoresOtherLists(t *testing.T) { + s := newTestStore(t) + a, _ := s.CreateList("a", "example.org", "A") + b, _ := s.CreateList("b", "example.org", "B") + s.ArchiveMessage(a.ID, msg("", "Topic", "r")) + reply := msg("", "Re: Topic", "r") + reply.InReplyTo = "" + id, _, _ := s.ArchiveMessage(b.ID, reply) + if m, _ := s.GetMessage(id); m.ThreadID != id { + t.Fatal("a parent in another list must not be used") + } +} + +func TestListMessagesNewestFirstWithPaging(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i := 0; i < 5; i++ { + m := msg("", "S"+string(rune('a'+i)), "body") + m.Date = time.Unix(1_700_000_000+int64(i), 0) + s.ArchiveMessage(l.ID, m) + } + got, total, err := s.ListMessages(l.ID, 2, 0) + if err != nil || total != 5 || len(got) != 2 { + t.Fatalf("page 1: %d %d %v", len(got), total, err) + } + if got[0].Subject != "Se" || got[1].Subject != "Sd" { + t.Fatalf("order: %s %s", got[0].Subject, got[1].Subject) + } + if got[0].Body != "" { + t.Fatal("listings must not load bodies") + } + got, _, _ = s.ListMessages(l.ID, 2, 4) + if len(got) != 1 || got[0].Subject != "Sa" { + t.Fatalf("last page: %+v", got) + } +} + +func TestSearch(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + other, _ := s.CreateList("ops", "example.org", "Ops") + s.ArchiveMessage(l.ID, msg("<1@x>", "Deploy failed", "the kubernetes rollout broke")) + s.ArchiveMessage(l.ID, msg("<2@x>", "Lunch", "pizza on friday")) + s.ArchiveMessage(other.ID, msg("<3@x>", "Deploy ok", "kubernetes is fine here")) + + got, total, err := s.SearchMessages(l.ID, "kubernetes", 10, 0) + if err != nil || total != 1 || len(got) != 1 || got[0].Subject != "Deploy failed" { + t.Fatalf("search: %+v %d %v", got, total, err) + } + // Subject, sender and multiple terms (AND). + if got, _, _ := s.SearchMessages(l.ID, "lunch pizza", 10, 0); len(got) != 1 { + t.Fatalf("subject+body: %+v", got) + } + if got, _, _ := s.SearchMessages(l.ID, "alice", 10, 0); len(got) != 2 { + t.Fatalf("sender: %+v", got) + } + if got, _, _ := s.SearchMessages(l.ID, "lunch kubernetes", 10, 0); len(got) != 0 { + t.Fatalf("AND semantics: %+v", got) + } +} + +func TestSearchIsSafeAgainstSyntax(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "world")) + for _, q := range []string{`"`, `foo AND`, `NEAR(`, `*`, `a OR b`, `col:x`, `)(`, "", " "} { + if _, _, err := s.SearchMessages(l.ID, q, 10, 0); err != nil { + t.Errorf("query %q must not error: %v", q, err) + } + } + // FTS operators are plain words: "world OR zzz" needs both terms. + if got, _, _ := s.SearchMessages(l.ID, "world OR zzz", 10, 0); len(got) != 0 { + t.Fatalf("operators must be literal: %+v", got) + } +} + +func TestDeleteMessageRemovesFromSearch(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + id, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Secret", "leaked password")) + if err := s.DeleteMessage(id); err != nil { + t.Fatal(err) + } + if got, _, _ := s.SearchMessages(l.ID, "password", 10, 0); len(got) != 0 { + t.Fatalf("deleted message still searchable: %+v", got) + } + if err := s.DeleteMessage(id); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestDeleteListCleansArchiveAndIndex(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "uniqueword")) + if err := s.DeleteList(l.ID); err != nil { + t.Fatal(err) + } + var n int + s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts WHERE archive_fts MATCH 'uniqueword'`).Scan(&n) + if n != 0 { + t.Fatal("search index must follow cascade deletes") + } +} + +func TestMigrationAddsArchiveColumn(t *testing.T) { + // A database from milestone 2 has no archive_policy column. + path := t.TempDir() + "/m2.db" + s, _ := Open(path) + s.CreateList("dev", "example.org", "Dev") + s.db.Exec(`ALTER TABLE lists DROP COLUMN archive_policy`) + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + l, err := s2.FindList("dev@example.org") + if err != nil || l.ArchivePolicy != ArchivePrivate { + t.Fatalf("list: %+v %v", l, err) + } +} + +func TestLongBodyIsStoredAsIs(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + body := strings.Repeat("x", 100_000) + id, _, _ := s.ArchiveMessage(l.ID, msg("", "Big", body)) + if got, _ := s.GetMessage(id); len(got.Body) != len(body) { + t.Fatalf("body len %d", len(got.Body)) + } +} diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 0e49708..1ca622d 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -14,17 +14,17 @@ func TestListPolicies(t *testing.T) { if l.PostPolicy != "members" || l.JoinPolicy != "open" { t.Fatalf("defaults: %+v", l) } - if err := s.SetPolicies(l.ID, "moderated", "closed"); err != nil { + if err := s.SetPolicies(l.ID, "moderated", "closed", "private"); err != nil { t.Fatal(err) } got, _ := s.GetList(l.ID) if got.PostPolicy != "moderated" || got.JoinPolicy != "closed" { t.Fatalf("policies: %+v", got) } - if err := s.SetPolicies(l.ID, "bogus", "open"); err == nil { + if err := s.SetPolicies(l.ID, "bogus", "open", "private"); err == nil { t.Fatal("invalid policy must be rejected") } - if err := s.SetPolicies(999, "members", "open"); !errors.Is(err, ErrNotFound) { + if err := s.SetPolicies(999, "members", "open", "private"); !errors.Is(err, ErrNotFound) { t.Fatalf("want ErrNotFound, got %v", err) } } diff --git a/internal/store/store.go b/internal/store/store.go index e74e487..4572fc4 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -28,6 +28,13 @@ const ( JoinClosed = "closed" // only the admin adds members ) +// Archive policy values. +const ( + ArchivePublic = "public" // browsable on the web + ArchivePrivate = "private" // stored, readable through the API only + ArchiveOff = "off" // not stored +) + // Pending actions. const ( ActionJoin = "join" @@ -41,12 +48,13 @@ const ( // List is a mailing list. type List struct { - ID int64 - Name string // local part, lowercase - Domain string // lowercase - Title string - PostPolicy string - JoinPolicy string + ID int64 + Name string // local part, lowercase + Domain string // lowercase + Title string + PostPolicy string + JoinPolicy string + ArchivePolicy string } // Address returns the list posting address. @@ -94,6 +102,7 @@ CREATE TABLE IF NOT EXISTS lists ( title TEXT NOT NULL DEFAULT '', post_policy TEXT NOT NULL DEFAULT 'members', join_policy TEXT NOT NULL DEFAULT 'open', + archive_policy TEXT NOT NULL DEFAULT 'private', UNIQUE(name, domain) ); CREATE TABLE IF NOT EXISTS members ( @@ -119,6 +128,30 @@ CREATE TABLE IF NOT EXISTS pending ( action TEXT NOT NULL, created_at INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS archive ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + thread_id INTEGER NOT NULL DEFAULT 0, + message_id TEXT NOT NULL, + in_reply_to TEXT NOT NULL DEFAULT '', + subject TEXT NOT NULL DEFAULT '', + sender TEXT NOT NULL DEFAULT '', + date INTEGER NOT NULL, + body TEXT NOT NULL DEFAULT '', + size INTEGER NOT NULL DEFAULT 0, + UNIQUE(list_id, message_id) +); +CREATE INDEX IF NOT EXISTS archive_list_date ON archive(list_id, date DESC, id DESC); +CREATE INDEX IF NOT EXISTS archive_thread ON archive(list_id, thread_id); +CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5( + subject, body, sender, content='archive', content_rowid='id' +); +CREATE TRIGGER IF NOT EXISTS archive_ai AFTER INSERT ON archive BEGIN + INSERT INTO archive_fts(rowid, subject, body, sender) VALUES (new.id, new.subject, new.body, new.sender); +END; +CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN + INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender); +END; CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -180,6 +213,15 @@ func migrate(db *sql.DB) error { } } } + if ok, err := hasColumn(db, "lists", "name"); err != nil { + return err + } else if ok { + if has, _ := hasColumn(db, "lists", "archive_policy"); !has { + if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN archive_policy TEXT NOT NULL DEFAULT 'private'`); err != nil { + return err + } + } + } _, err := db.Exec(schema) return err } @@ -203,11 +245,11 @@ func affected(res sql.Result, err error) error { // Lists. -const listCols = `id, name, domain, title, post_policy, join_policy` +const listCols = `id, name, domain, title, post_policy, join_policy, archive_policy` func scanList(row interface{ Scan(...any) error }) (List, error) { var l List - err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy) + err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy, &l.ArchivePolicy) if errors.Is(err, sql.ErrNoRows) { return List{}, ErrNotFound } @@ -265,15 +307,18 @@ func (s *Store) UpdateList(id int64, title string) error { return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id)) } -// SetPolicies changes the post and join policies of a list. -func (s *Store) SetPolicies(id int64, post, join string) error { +// SetPolicies changes the post, join and archive policies of a list. +func (s *Store) SetPolicies(id int64, post, join, archive string) error { if post != PostMembers && post != PostModerated { return fmt.Errorf("invalid post policy %q", post) } if join != JoinOpen && join != JoinClosed { return fmt.Errorf("invalid join policy %q", join) } - return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=? WHERE id=?`, post, join, id)) + if archive != ArchivePublic && archive != ArchivePrivate && archive != ArchiveOff { + return fmt.Errorf("invalid archive policy %q", archive) + } + return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=?, archive_policy=? WHERE id=?`, post, join, archive, id)) } // DeleteList removes a list with its members, held messages and pending requests. @@ -349,6 +394,11 @@ func (s *Store) RemoveMember(listID int64, addr string) error { return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr))) } +// DeleteMember unsubscribes a member by id. +func (s *Store) DeleteMember(id int64) error { + return affected(s.db.Exec(`DELETE FROM members WHERE id=?`, id)) +} + // RecordHardBounce counts a hard bounce and disables the member once the count // reaches threshold. It reports whether the member is now disabled. func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) { @@ -485,3 +535,159 @@ func (s *Store) Secret() ([]byte, error) { } return hex.DecodeString(v) } + +// Archive. + +// Message is an archived post. +type Message struct { + ID int64 + ListID int64 + ThreadID int64 + MessageID string // Message-ID header, unique per list + InReplyTo string + Subject string + Sender string // display name of the original author + Date time.Time + Body string // plain text; empty in listings + Size int +} + +const msgCols = `id, list_id, thread_id, message_id, in_reply_to, subject, sender, date, size` + +func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, error) { + var m Message + var ts int64 + var err error + if withBody { + err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size, &m.Body) + } else { + err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size) + } + if errors.Is(err, sql.ErrNoRows) { + return Message{}, ErrNotFound + } + m.Date = time.Unix(ts, 0) + return m, err +} + +// ArchiveMessage stores a post. A message already archived in the same list +// (same Message-ID) is not stored again: created is false and the existing id +// is returned, so MTA retries are harmless. +func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) { + if m.MessageID == "" { + return 0, false, errors.New("message id is required") + } + err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id) + if err == nil { + return id, false, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, false, err + } + var thread int64 + if m.InReplyTo != "" { + e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread) + if e != nil && !errors.Is(e, sql.ErrNoRows) { + return 0, false, e + } + } + res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size) + VALUES(?,?,?,?,?,?,?,?,?)`, + listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body)) + if err != nil { + return 0, false, err + } + id, _ = res.LastInsertId() + if thread == 0 { + if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil { + return 0, false, err + } + } + return id, true, nil +} + +// GetMessage returns an archived message with its body. +func (s *Store) GetMessage(id int64) (Message, error) { + return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true) +} + +// DeleteMessage removes an archived message and its search entry. +func (s *Store) DeleteMessage(id int64) error { + return affected(s.db.Exec(`DELETE FROM archive WHERE id=?`, id)) +} + +func (s *Store) queryMessages(q string, args ...any) ([]Message, error) { + rows, err := s.db.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Message + for rows.Next() { + m, err := scanMessage(rows, false) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} + +// ListMessages returns a page of a list's archive, newest first, and the total +// count. Bodies are not loaded. +func (s *Store) ListMessages(listID int64, limit, offset int) ([]Message, int, error) { + var total int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive WHERE list_id=?`, listID).Scan(&total); err != nil { + return nil, 0, err + } + out, err := s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? ORDER BY date DESC, id DESC LIMIT ? OFFSET ?`, + listID, limit, offset) + return out, total, err +} + +// ftsQuery turns free text into an FTS5 query where every word is a quoted +// literal (AND semantics), so user input can never be read as FTS syntax. +func ftsQuery(q string) string { + words := strings.Fields(q) + for i, w := range words { + words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"` + } + return strings.Join(words, " ") +} + +// SearchMessages runs a full-text search over subject, body and sender of one +// list, newest first. Bodies are not loaded. +func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([]Message, int, error) { + match := ftsQuery(query) + if match == "" { + return nil, 0, nil + } + var total int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid + WHERE archive_fts MATCH ? AND a.list_id=?`, match, listID).Scan(&total); err != nil { + return nil, 0, err + } + out, err := s.queryMessages(`SELECT a.id, a.list_id, a.thread_id, a.message_id, a.in_reply_to, a.subject, a.sender, a.date, a.size + FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid + WHERE archive_fts MATCH ? AND a.list_id=? ORDER BY a.date DESC, a.id DESC LIMIT ? OFFSET ?`, + match, listID, limit, offset) + return out, total, err +} + +// ThreadMessages returns every message of a thread, oldest first, with bodies. +func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) { + rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Message + for rows.Next() { + m, err := scanMessage(rows, true) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} diff --git a/internal/web/web.go b/internal/web/web.go new file mode 100644 index 0000000..53d36dc --- /dev/null +++ b/internal/web/web.go @@ -0,0 +1,299 @@ +// Package web serves the public archive pages and the unsubscribe links. +package web + +import ( + "bytes" + "errors" + "html/template" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "codeberg.org/squidd/parsimony/internal/archive" + "codeberg.org/squidd/parsimony/internal/router" + "codeberg.org/squidd/parsimony/internal/store" +) + +const pageSize = 25 + +type server struct { + store *store.Store + router *router.Router + tmpl *template.Template +} + +// New returns the public web handler. +func New(s *store.Store, rt *router.Router) http.Handler { + srv := &server{store: s, router: rt, tmpl: template.Must(template.New("").Funcs(template.FuncMap{ + "hide": archive.Obfuscate, + "date": func(t time.Time) string { return t.UTC().Format("2006-01-02 15:04") }, + }).Parse(templates))} + + mux := http.NewServeMux() + mux.HandleFunc("GET /{$}", srv.index) + mux.HandleFunc("GET /archive/{address}", srv.archive) + mux.HandleFunc("GET /archive/{address}/{id}", srv.message) + mux.HandleFunc("GET /unsubscribe/{token}", srv.unsubscribePage) + mux.HandleFunc("POST /unsubscribe/{token}", srv.unsubscribe) + // Catch-all for GET so unknown paths get the HTML 404 page and other + // methods on known paths still answer 405. + mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { srv.notFound(w) }) + return secure(mux) +} + +// secure adds the headers every page needs. +func secure(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'") + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Referrer-Policy", "no-referrer") + h.Set("Content-Type", "text/html; charset=utf-8") + if strings.HasPrefix(r.URL.Path, "/unsubscribe/") { + h.Set("Cache-Control", "no-store") // the URL carries a secret + } + next.ServeHTTP(w, r) + }) +} + +func (s *server) render(w http.ResponseWriter, status int, name string, data any) { + var buf bytes.Buffer + if err := s.tmpl.ExecuteTemplate(&buf, name, data); err != nil { + log.Printf("web: render %s: %v", name, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + w.WriteHeader(status) + w.Write(buf.Bytes()) +} + +func (s *server) notFound(w http.ResponseWriter) { s.render(w, http.StatusNotFound, "notfound", nil) } + +func (s *server) fail(w http.ResponseWriter, err error) { + log.Printf("web: %v", err) + http.Error(w, "internal error", http.StatusInternalServerError) +} + +// publicList resolves a list address and only returns lists with a public archive. +func (s *server) publicList(address string) (store.List, bool, error) { + l, err := s.store.FindList(address) + if errors.Is(err, store.ErrNotFound) || (err == nil && l.ArchivePolicy != store.ArchivePublic) { + return store.List{}, false, nil + } + return l, err == nil, err +} + +func (s *server) index(w http.ResponseWriter, r *http.Request) { + all, err := s.store.ListLists() + if err != nil { + s.fail(w, err) + return + } + var lists []store.List + for _, l := range all { + if l.ArchivePolicy == store.ArchivePublic { + lists = append(lists, l) + } + } + s.render(w, http.StatusOK, "index", lists) +} + +type archiveView struct { + List store.List + Query string + Messages []store.Message + Total int + Page int + PrevURL string + NextURL string + ResultText string +} + +func (s *server) archive(w http.ResponseWriter, r *http.Request) { + l, ok, err := s.publicList(r.PathValue("address")) + if err != nil { + s.fail(w, err) + return + } + if !ok { + s.notFound(w) + return + } + q := strings.TrimSpace(r.URL.Query().Get("q")) + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + page = max(page, 1) + + fetch := func(offset int) ([]store.Message, int, error) { + if q != "" { + return s.store.SearchMessages(l.ID, q, pageSize, offset) + } + return s.store.ListMessages(l.ID, pageSize, offset) + } + ms, total, err := fetch((page - 1) * pageSize) + if err == nil { + if pages := (total + pageSize - 1) / pageSize; pages > 0 && page > pages { + page = pages + ms, total, err = fetch((page - 1) * pageSize) + } + } + if err != nil { + s.fail(w, err) + return + } + + v := archiveView{List: l, Query: q, Messages: ms, Total: total, Page: page} + link := func(p int) string { + vals := url.Values{} + if q != "" { + vals.Set("q", q) + } + if p > 1 { + vals.Set("page", strconv.Itoa(p)) + } + u := "/archive/" + l.Address() + if len(vals) > 0 { + u += "?" + vals.Encode() + } + return u + } + if page > 1 { + v.PrevURL = link(page - 1) + } + if page*pageSize < total { + v.NextURL = link(page + 1) + } + if q != "" { + v.ResultText = strconv.Itoa(total) + " results" + if total == 1 { + v.ResultText = "1 result" + } + } + s.render(w, http.StatusOK, "archive", v) +} + +type messageView struct { + List store.List + Message store.Message + Thread []store.Message +} + +func (s *server) message(w http.ResponseWriter, r *http.Request) { + l, ok, err := s.publicList(r.PathValue("address")) + if err != nil { + s.fail(w, err) + return + } + id, perr := strconv.ParseInt(r.PathValue("id"), 10, 64) + if !ok || perr != nil || id <= 0 { + s.notFound(w) + return + } + m, err := s.store.GetMessage(id) + if errors.Is(err, store.ErrNotFound) || (err == nil && m.ListID != l.ID) { + s.notFound(w) // also hides messages that belong to other lists + return + } + if err != nil { + s.fail(w, err) + return + } + thread, err := s.store.ThreadMessages(l.ID, m.ThreadID) + if err != nil { + s.fail(w, err) + return + } + s.render(w, http.StatusOK, "message", messageView{List: l, Message: m, Thread: thread}) +} + +func (s *server) unsubscribePage(w http.ResponseWriter, r *http.Request) { + l, err := s.router.CheckUnsubscribe(r.PathValue("token")) + switch { + case errors.Is(err, router.ErrBadToken): + s.notFound(w) + case errors.Is(err, router.ErrGone): + s.render(w, http.StatusOK, "unsubscribed", map[string]any{"Already": true}) + case err != nil: + s.fail(w, err) + default: + s.render(w, http.StatusOK, "unsubscribe", l) + } +} + +func (s *server) unsubscribe(w http.ResponseWriter, r *http.Request) { + err := s.router.Unsubscribe(r.PathValue("token")) + switch { + case errors.Is(err, router.ErrBadToken): + s.notFound(w) + case err != nil: + s.fail(w, err) + default: + s.render(w, http.StatusOK, "unsubscribed", map[string]any{"Already": false}) + } +} + +const templates = ` +{{define "head"}} + +{{.}} +{{end}} +{{define "foot"}}{{end}} + +{{define "index"}}{{template "head" "Mailing list archives"}} +

Mailing list archives

+{{if .}} +{{else}}

No public archives.

{{end}} +{{template "foot"}}{{end}} + +{{define "archive"}}{{template "head" .List.Title}} + +

{{.List.Title}}

+

{{.List.Address}}

+
+{{if .Query}}

{{.ResultText}} for "{{.Query}}"

{{end}} +{{if .Messages}} +{{range .Messages}} +{{end}}
SubjectFromDate
{{hide .Subject}}{{hide .Sender}}{{date .Date}}
+

{{if .PrevURL}}← Newer{{end}} {{if .NextURL}}Older →{{end}}

+{{else}}

{{if .Query}}No messages found.{{else}}No messages yet.{{end}}

{{end}} +{{template "foot"}}{{end}} + +{{define "message"}}{{template "head" (hide .Message.Subject)}} + +

{{hide .Message.Subject}}

+

{{hide .Message.Sender}} · {{date .Message.Date}}

+
{{hide .Message.Body}}
+{{if gt (len .Thread) 1}}

Thread

    +{{range .Thread}}
  • {{if eq .ID $.Message.ID}}{{hide .Subject}}{{else}}{{hide .Subject}}{{end}} +{{hide .Sender}}, {{date .Date}}
  • {{end}}
{{end}} +{{template "foot"}}{{end}} + +{{define "unsubscribe"}}{{template "head" "Unsubscribe"}} +

Unsubscribe

+

Stop receiving messages from {{.Title}} ({{.Address}})?

+
+{{template "foot"}}{{end}} + +{{define "unsubscribed"}}{{template "head" "Unsubscribed"}} +

{{if .Already}}You are already unsubscribed{{else}}You are unsubscribed{{end}}

+

{{if .Already}}This address no longer receives messages from the list.{{else}}You will not receive any more messages from the list.{{end}}

+{{template "foot"}}{{end}} + +{{define "notfound"}}{{template "head" "Not found"}} +

Not found

Back to the archives

+{{template "foot"}}{{end}} +` diff --git a/internal/web/web_test.go b/internal/web/web_test.go new file mode 100644 index 0000000..bbdd4c4 --- /dev/null +++ b/internal/web/web_test.go @@ -0,0 +1,291 @@ +package web + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "codeberg.org/squidd/parsimony/internal/router" + "codeberg.org/squidd/parsimony/internal/store" +) + +type fixture struct { + h http.Handler + s *store.Store + rt *router.Router + pub store.List + alice int64 + bobTok string +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + s, err := store.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + pub, _ := s.CreateList("dev", "example.org", "Dev talk") + s.SetPolicies(pub.ID, store.PostMembers, store.JoinOpen, store.ArchivePublic) + priv, _ := s.CreateList("secret", "example.org", "Secret") + s.SetPolicies(priv.ID, store.PostMembers, store.JoinOpen, store.ArchivePrivate) + off, _ := s.CreateList("off", "example.org", "Off") + s.SetPolicies(off.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff) + + s.AddMember(pub.ID, "alice@example.com", "Alice") + s.AddMember(pub.ID, "bob@example.com", "Bob") + ms, _ := s.Members(pub.ID) + rt := router.New(s) + return &fixture{h: New(s, rt), s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID)} +} + +func (f *fixture) add(t *testing.T, listID int64, id, subject, sender, body string, at int64, inReplyTo string) int64 { + t.Helper() + n, _, err := f.s.ArchiveMessage(listID, store.Message{ + MessageID: id, InReplyTo: inReplyTo, Subject: subject, Sender: sender, Body: body, Date: time.Unix(at, 0).UTC(), + }) + if err != nil { + t.Fatal(err) + } + return n +} + +func get(t *testing.T, h http.Handler, path string) *httptest.ResponseRecorder { + t.Helper() + return request(t, h, "GET", path) +} + +func request(t *testing.T, h http.Handler, method, path string) *httptest.ResponseRecorder { + t.Helper() + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(method, path, nil)) + return rec +} + +func wantStatus(t *testing.T, rec *httptest.ResponseRecorder, code int) { + t.Helper() + if rec.Code != code { + t.Fatalf("status = %d, want %d; body: %.300s", rec.Code, code, rec.Body.String()) + } +} + +func wantBody(t *testing.T, rec *httptest.ResponseRecorder, parts ...string) { + t.Helper() + for _, p := range parts { + if !strings.Contains(rec.Body.String(), p) { + t.Errorf("body lacks %q:\n%.800s", p, rec.Body.String()) + } + } +} + +func wantNoBody(t *testing.T, rec *httptest.ResponseRecorder, parts ...string) { + t.Helper() + for _, p := range parts { + if strings.Contains(rec.Body.String(), p) { + t.Errorf("body must not contain %q", p) + } + } +} + +func TestIndexListsOnlyPublicLists(t *testing.T) { + f := newFixture(t) + rec := get(t, f.h, "/") + wantStatus(t, rec, 200) + wantBody(t, rec, "dev@example.org", `href="/archive/dev@example.org"`) + wantNoBody(t, rec, "secret@example.org", "off@example.org") +} + +func TestIndexEscapesTitles(t *testing.T) { + f := newFixture(t) + rec := get(t, f.h, "/") + wantBody(t, rec, "Dev <b>talk</b>") + wantNoBody(t, rec, "talk") +} + +func TestIndexWithoutPublicLists(t *testing.T) { + s, _ := store.Open(":memory:") + defer s.Close() + rec := get(t, New(s, router.New(s)), "/") + wantStatus(t, rec, 200) + wantBody(t, rec, "No public archives") +} + +func TestUnknownPathIs404(t *testing.T) { + f := newFixture(t) + wantStatus(t, get(t, f.h, "/nope"), 404) +} + +func TestNonPublicOrUnknownArchivesAre404(t *testing.T) { + f := newFixture(t) + for _, p := range []string{"/archive/secret@example.org", "/archive/off@example.org", "/archive/ghost@example.org", + "/archive/not-an-address", "/archive/secret@example.org?q=x"} { + wantStatus(t, get(t, f.h, p), 404) + } +} + +func TestArchivePageShowsMessagesNewestFirst(t *testing.T) { + f := newFixture(t) + f.add(t, f.pub.ID, "<1@x>", "Old topic", "Alice", "a", 1_700_000_000, "") + f.add(t, f.pub.ID, "<2@x>", "New topic", "Bob ", "b", 1_700_100_000, "") + rec := get(t, f.h, "/archive/dev@example.org") + wantStatus(t, rec, 200) + wantBody(t, rec, "Old topic", "New topic", "Bob <i>", "2023-11-14") + wantNoBody(t, rec, "Bob ") + if strings.Index(rec.Body.String(), "New topic") > strings.Index(rec.Body.String(), "Old topic") { + t.Fatal("newest message must come first") + } +} + +func TestArchivePaging(t *testing.T) { + f := newFixture(t) + for i := 0; i < pageSize+3; i++ { + f.add(t, f.pub.ID, fmt.Sprintf("", i), fmt.Sprintf("Subject number %d", i), "A", "b", 1_700_000_000+int64(i), "") + } + p1 := get(t, f.h, "/archive/dev@example.org") + wantBody(t, p1, fmt.Sprintf("Subject number %d", pageSize+2), "page=2") + wantNoBody(t, p1, "Subject number 0<", "page=0") + p2 := get(t, f.h, "/archive/dev@example.org?page=2") + wantStatus(t, p2, 200) + wantBody(t, p2, "Subject number 0<") + wantNoBody(t, p2, "page=3") + // A page past the end shows the last page instead of an empty one. + far := get(t, f.h, "/archive/dev@example.org?page=999") + wantBody(t, far, "Subject number 0<") + for _, bad := range []string{"?page=abc", "?page=0", "?page=-4", "?page=999"} { + wantStatus(t, get(t, f.h, "/archive/dev@example.org"+bad), 200) + } +} + +func TestArchiveSearch(t *testing.T) { + f := newFixture(t) + f.add(t, f.pub.ID, "<1@x>", "Deploy", "A", "kubernetes rollout", 1_700_000_000, "") + f.add(t, f.pub.ID, "<2@x>", "Lunch", "B", "pizza", 1_700_000_001, "") + rec := get(t, f.h, "/archive/dev@example.org?q=kubernetes") + wantStatus(t, rec, 200) + wantBody(t, rec, "Deploy", "1 result") + wantNoBody(t, rec, "Lunch") + rec = get(t, f.h, "/archive/dev@example.org?q=zzzz") + wantBody(t, rec, "No messages found") + // The query is echoed back escaped and survives paging links. + rec = get(t, f.h, "/archive/dev@example.org?q="+url.QueryEscape(`">`)) + wantStatus(t, rec, 200) + wantNoBody(t, rec, "") +} + +func TestMessagePage(t *testing.T) { + f := newFixture(t) + root := f.add(t, f.pub.ID, "", "Original topic", "Alice", "Write to carol@example.com ", 1_700_000_000, "") + reply := f.add(t, f.pub.ID, "", "Re: Original topic", "Bob", "I agree", 1_700_000_100, "") + + rec := get(t, f.h, fmt.Sprintf("/archive/dev@example.org/%d", root)) + wantStatus(t, rec, 200) + wantBody(t, rec, "Original topic", "Alice", "carol@…", "<script>alert(1)</script>", + fmt.Sprintf(`href="/archive/dev@example.org/%d"`, reply), "Re: Original topic") + wantNoBody(t, rec, "carol@example.com", "") + + rec = get(t, f.h, fmt.Sprintf("/archive/dev@example.org/%d", reply)) + wantStatus(t, rec, 200) + wantBody(t, rec, "I agree", fmt.Sprintf(`href="/archive/dev@example.org/%d"`, root)) +} + +func TestMessagePageHidesOtherListsAndPrivateOnes(t *testing.T) { + f := newFixture(t) + priv, _ := f.s.FindList("secret@example.org") + secretID := f.add(t, priv.ID, "", "Top secret", "A", "classified", 1_700_000_000, "") + pubID := f.add(t, f.pub.ID, "", "Public", "A", "hello", 1_700_000_000, "") + + wantStatus(t, get(t, f.h, fmt.Sprintf("/archive/secret@example.org/%d", secretID)), 404) + // A public list URL must not reveal a message that belongs to another list. + wantStatus(t, get(t, f.h, fmt.Sprintf("/archive/dev@example.org/%d", secretID)), 404) + wantStatus(t, get(t, f.h, fmt.Sprintf("/archive/dev@example.org/%d", pubID)), 200) + for _, p := range []string{"/archive/dev@example.org/abc", "/archive/dev@example.org/0", "/archive/dev@example.org/999", "/archive/dev@example.org/-1"} { + wantStatus(t, get(t, f.h, p), 404) + } +} + +func TestPrivacyOfOriginalSender(t *testing.T) { + f := newFixture(t) + id := f.add(t, f.pub.ID, "<1@x>", "Contact bob@secret.example", "bob", "ping", 1_700_000_000, "") + rec := get(t, f.h, fmt.Sprintf("/archive/dev@example.org/%d", id)) + wantNoBody(t, rec, "bob@secret.example") + wantBody(t, rec, "bob@…") + rec = get(t, f.h, "/archive/dev@example.org") + wantNoBody(t, rec, "bob@secret.example") +} + +func TestSecurityHeaders(t *testing.T) { + f := newFixture(t) + for _, p := range []string{"/", "/archive/dev@example.org", "/nope"} { + h := get(t, f.h, p).Header() + if h.Get("X-Content-Type-Options") != "nosniff" || h.Get("Referrer-Policy") != "no-referrer" || + !strings.Contains(h.Get("Content-Security-Policy"), "default-src 'none'") || + !strings.HasPrefix(h.Get("Content-Type"), "text/html") { + t.Errorf("%s: headers %v", p, h) + } + } +} + +func TestOnlyGetOnArchive(t *testing.T) { + f := newFixture(t) + wantStatus(t, request(t, f.h, "POST", "/archive/dev@example.org"), 405) + wantStatus(t, request(t, f.h, "DELETE", "/"), 405) +} + +func TestUnsubscribeConfirmPageDoesNotUnsubscribe(t *testing.T) { + f := newFixture(t) + rec := get(t, f.h, "/unsubscribe/"+f.bobTok) + wantStatus(t, rec, 200) + wantBody(t, rec, `method="post"`, "Dev <b>talk</b>") + if ok, _ := f.s.IsMember(f.pub.ID, "bob@example.com"); !ok { + t.Fatal("a GET (link prefetch) must never unsubscribe") + } + h := rec.Header() + if h.Get("Cache-Control") != "no-store" || h.Get("Referrer-Policy") != "no-referrer" { + t.Errorf("token page headers: %v", h) + } +} + +func TestUnsubscribePost(t *testing.T) { + f := newFixture(t) + rec := request(t, f.h, "POST", "/unsubscribe/"+f.bobTok) + wantStatus(t, rec, 200) + wantBody(t, rec, "unsubscribed") + if ok, _ := f.s.IsMember(f.pub.ID, "bob@example.com"); ok { + t.Fatal("bob must be unsubscribed") + } + if ok, _ := f.s.IsMember(f.pub.ID, "alice@example.com"); !ok { + t.Fatal("alice must stay") + } + // Repeated one-click requests and a later visit are harmless. + wantStatus(t, request(t, f.h, "POST", "/unsubscribe/"+f.bobTok), 200) + rec = get(t, f.h, "/unsubscribe/"+f.bobTok) + wantStatus(t, rec, 200) + wantBody(t, rec, "already unsubscribed") +} + +func TestUnsubscribeRejectsBadTokens(t *testing.T) { + f := newFixture(t) + for _, tok := range []string{"x", "1-000000000000", "abc-def", f.bobTok + "0", "0-" + strings.Repeat("0", 12)} { + wantStatus(t, get(t, f.h, "/unsubscribe/"+tok), 404) + wantStatus(t, request(t, f.h, "POST", "/unsubscribe/"+tok), 404) + } + if n, _ := f.s.Members(f.pub.ID); len(n) != 2 { + t.Fatal("bad tokens must not remove anyone") + } +} + +func TestUnsubscribeOneClickBodyIsAccepted(t *testing.T) { + f := newFixture(t) + req := httptest.NewRequest("POST", "/unsubscribe/"+f.bobTok, strings.NewReader("List-Unsubscribe=One-Click")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rec := httptest.NewRecorder() + f.h.ServeHTTP(rec, req) + wantStatus(t, rec, 200) + if ok, _ := f.s.IsMember(f.pub.ID, "bob@example.com"); ok { + t.Fatal("bob must be unsubscribed") + } +} diff --git a/parsimony.example.yaml b/parsimony.example.yaml index 7240253..baea5be 100644 --- a/parsimony.example.yaml +++ b/parsimony.example.yaml @@ -5,6 +5,8 @@ lmtp: addr: 0.0.0.0:2424 relay: addr: postfix:25 +web: + listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them api: listen: 0.0.0.0:8080 # The API token is read from the PARSIMONY_API_TOKEN environment variable.