diff --git a/README.md b/README.md index f3ab39e..2d989c8 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ A modern, single-binary mailing list manager: an evolution of Mailman. send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. - Moderated lists: posts wait in a queue that the API can approve or discard. - VERP bounces: every recipient gets a signed return address; 3 hard bounces disable a member. +- Archive with full-text search (SQLite FTS5) and threading. Each list has an + `archive_policy`: `private` (default, API only), `public` (web pages) or `off`. + Public pages hide email addresses. +- Per-recipient one-click unsubscribe links (RFC 8058) with signed tokens. - SQLite storage, one YAML config file. - Admin REST API under `/api/v1` (see `docs/openapi.yaml`). @@ -21,6 +25,17 @@ PARSIMONY_API_TOKEN= go run ./cmd/parsimony -config parsimony.yaml The API is disabled when `PARSIMONY_API_TOKEN` is not set. +Three listeners, each configurable: + +| Listener | Default | Purpose | +|----------|---------|---------| +| `lmtp` | `127.0.0.1:2424` | mail from Postfix | +| `api` | `127.0.0.1:8080` | admin REST API, bearer token | +| `web` | `127.0.0.1:8081` | public archive and unsubscribe pages (`""` disables) | + +Put the `web` listener behind a reverse proxy with TLS and set `base_url` to its public URL: +the unsubscribe links in every message use it. Keep the `api` listener private. + ``` curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists ``` diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 16c2b82..e5d16b8 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -22,6 +22,19 @@ components: address: {type: string} post_policy: {type: string, enum: [members, moderated]} join_policy: {type: string, enum: [open, closed]} + archive_policy: {type: string, enum: [public, private, off], description: "public: browsable on the web; private: API only (default); off: not stored"} + Message: + type: object + properties: + id: {type: integer} + thread_id: {type: integer} + message_id: {type: string} + in_reply_to: {type: string} + subject: {type: string} + sender: {type: string, description: Display name of the author} + date: {type: string, format: date-time} + size: {type: integer} + body: {type: string, description: Only when reading a single message} Member: type: object properties: @@ -108,6 +121,7 @@ paths: title: {type: string} post_policy: {type: string, enum: [members, moderated]} join_policy: {type: string, enum: [open, closed]} + archive_policy: {type: string, enum: [public, private, off]} responses: '200': description: OK @@ -191,3 +205,41 @@ paths: responses: '204': {description: Discarded} '404': {$ref: '#/components/responses/Error'} + /lists/{id}/archive: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + get: + summary: List or search archived messages, newest first + parameters: + - {name: q, in: query, schema: {type: string}, description: Full-text search over subject, body and sender; words are ANDed} + - {name: limit, in: query, schema: {type: integer, default: 50, maximum: 200}} + - {name: offset, in: query, schema: {type: integer, default: 0}} + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + total: {type: integer} + messages: {type: array, items: {$ref: '#/components/schemas/Message'}} + '400': {$ref: '#/components/responses/Error'} + '404': {$ref: '#/components/responses/Error'} + /archive/{id}: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + get: + summary: Read an archived message with its body + responses: + '200': + description: OK + content: + application/json: + schema: {$ref: '#/components/schemas/Message'} + '404': {$ref: '#/components/responses/Error'} + delete: + summary: Delete an archived message and its search entry + responses: + '204': {description: Deleted} + '404': {$ref: '#/components/responses/Error'} diff --git a/internal/api/api_test.go b/internal/api/api_test.go index e7f6da9..0045542 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -396,8 +396,25 @@ func TestArchiveList(t *testing.T) { } func TestArchiveLimitIsCapped(t *testing.T) { - h := archiveServer(t) - expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=100000", "", true), 200) + s, _ := store.Open(":memory:") + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i := 0; i < maxPageSize+5; i++ { + s.ArchiveMessage(l.ID, store.Message{MessageID: fmt.Sprintf("<%d@x>", i), Subject: "s", Date: time.Unix(int64(i), 0)}) + } + h := New(s, token, nil) + rec := do(t, h, "GET", "/api/v1/lists/1/archive?limit=100000", "", true) + expect(t, rec, 200) + var out archiveJSON + decode(t, rec, &out) + if len(out.Messages) != maxPageSize || out.Total != maxPageSize+5 { + t.Fatalf("got %d messages, total %d", len(out.Messages), out.Total) + } + rec = do(t, h, "GET", "/api/v1/lists/1/archive?limit=0", "", true) + decode(t, rec, &out) + if len(out.Messages) != 1 { + t.Fatalf("limit=0 must still return one message, got %d", len(out.Messages)) + } } func TestArchiveGetAndDelete(t *testing.T) { diff --git a/internal/app/app.go b/internal/app/app.go index 07478ce..55fc292 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -20,6 +20,7 @@ import ( "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/router" "codeberg.org/squidd/parsimony/internal/store" + "codeberg.org/squidd/parsimony/internal/web" ) // App is a running daemon. @@ -28,11 +29,13 @@ type App struct { // APIAddr is empty when the API is disabled. LMTPAddr string APIAddr string + WebAddr string // empty when the public web is disabled store *store.Store lmtp *smtp.Server lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it web *http.Server + pub *http.Server wg sync.WaitGroup once sync.Once } @@ -83,6 +86,21 @@ func Start(cfg config.Config, apiToken string) (*App, error) { a.web = &http.Server{Handler: api.New(st, apiToken, releaser{rt, relay}), ReadHeaderTimeout: 10 * time.Second} } + var pubLn net.Listener + if cfg.Web.Listen != "" { + pubLn, err = net.Listen("tcp", cfg.Web.Listen) + if err != nil { + for _, ln := range a.lns { + ln.Close() + } + st.Close() + return nil, fmt.Errorf("listen web: %w", err) + } + a.WebAddr = pubLn.Addr().String() + a.lns = append(a.lns, pubLn) + a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second} + } + a.wg.Add(1) go func() { defer a.wg.Done() @@ -99,6 +117,15 @@ func Start(cfg config.Config, apiToken string) (*App, error) { } }() } + if a.pub != nil { + a.wg.Add(1) + go func() { + defer a.wg.Done() + if err := a.pub.Serve(pubLn); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + log.Printf("web: %v", err) + } + }() + } return a, nil } @@ -112,6 +139,9 @@ func (a *App) Close() { if a.web != nil { a.web.Close() } + if a.pub != nil { + a.pub.Close() + } a.wg.Wait() a.store.Close() }) diff --git a/internal/app/app_test.go b/internal/app/app_test.go index a984a65..020331b 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -58,6 +58,12 @@ func (s *sink) count() int { return len(s.msgs) } +func (s *sink) all() []sunk { + s.mu.Lock() + defer s.mu.Unlock() + return append([]sunk(nil), s.msgs...) +} + func startSink(t *testing.T) (string, *sink) { t.Helper() sk := &sink{} @@ -79,6 +85,7 @@ func testConfig(t *testing.T, relay string) config.Config { c.LMTP.Addr = "127.0.0.1:0" c.Relay.Addr = relay c.API.Listen = "127.0.0.1:0" + c.Web.Listen = "127.0.0.1:0" return c } @@ -96,6 +103,18 @@ func call(t *testing.T, a *App, method, path, body, token string) int { return resp.StatusCode } +func fetch(t *testing.T, method, url string) (int, string) { + t.Helper() + req, _ := http.NewRequest(method, url, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(b) +} + func lmtpSend(t *testing.T, addr, from, rcpt, msg string) error { t.Helper() conn, err := net.Dial("tcp", addr) @@ -210,3 +229,123 @@ func TestCloseIsIdempotentAndStopsListeners(t *testing.T) { t.Fatal("LMTP still accepting after Close") } } + +func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) { + relayAddr, relay := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.BaseURL = "http://placeholder.invalid" + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + if a.WebAddr == "" { + t.Fatal("web must be listening") + } + + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") + for _, m := range []string{"alice@example.com", "bob@example.com"} { + call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok") + } + web := "http://" + a.WebAddr + + // Private by default: stored but not on the web. + msg := "From: Alice \r\nSubject: Quarterly plan\r\nMessage-ID: \r\n\r\nmail me at alice@example.com\r\n" + if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { + t.Fatal(err) + } + if code, _ := fetch(t, "GET", web+"/archive/dev@example.org"); code != 404 { + t.Fatalf("private archive must be 404, got %d", code) + } + if code := call(t, a, "GET", "/api/v1/lists/1/archive?q=quarterly", "", "tok"); code != 200 { + t.Fatalf("api archive = %d", code) + } + + // Make it public. + if code := call(t, a, "PATCH", "/api/v1/lists/1", `{"archive_policy":"public"}`, "tok"); code != 200 { + t.Fatalf("patch = %d", code) + } + code, body := fetch(t, "GET", web+"/archive/dev@example.org?q=quarterly") + if code != 200 || !strings.Contains(body, "Quarterly plan") { + t.Fatalf("public archive: %d %.200s", code, body) + } + code, body = fetch(t, "GET", web+"/archive/dev@example.org/1") + if code != 200 || !strings.Contains(body, "alice@…") || strings.Contains(body, "alice@example.com") { + t.Fatalf("message page must hide addresses: %d %.300s", code, body) + } + + // The relayed copy carries a working one-click link for that recipient. + var link string + for _, m := range relay.all() { + if len(m.to) == 1 && m.to[0] == "bob@example.com" { + i := strings.Index(m.data, "List-Unsubscribe: ")] + } + } + if link == "" { + t.Fatal("bob got no copy") + } + if code, _ := fetch(t, "GET", web+link); code != 200 { + t.Fatalf("confirm page = %d", code) + } + if code, _ := fetch(t, "POST", web+link); code != 200 { + t.Fatalf("one-click = %d", code) + } + before := relay.count() + if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", + "From: Alice \r\nSubject: Second\r\nMessage-ID: \r\n\r\nhi\r\n"); err != nil { + t.Fatal(err) + } + if got := relay.count() - before; got != 1 { + t.Fatalf("after bob left only alice should get a copy, got %d", got) + } +} + +func TestWebDisabledWhenListenIsEmpty(t *testing.T) { + relayAddr, _ := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.Web.Listen = "" + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + if a.WebAddr != "" { + t.Fatalf("web must be off, listening on %s", a.WebAddr) + } +} + +func TestCloseDropsOpenConnections(t *testing.T) { + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + conns := map[string]net.Conn{} + for name, addr := range map[string]string{"web": a.WebAddr, "api": a.APIAddr, "lmtp": a.LMTPAddr} { + c, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + conns[name] = c + } + time.Sleep(100 * time.Millisecond) // let the servers accept + a.Close() + for name, c := range conns { + c.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 512) + for { + if _, err := c.Read(buf); err != nil { + if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Errorf("%s: connection still open after Close", name) + } + break + } + } + } +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index a2f14e6..fa96e29 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" "time" + "unicode/utf8" ) var now = time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC) @@ -182,3 +183,32 @@ func TestObfuscate(t *testing.T) { } } } + +func TestTextAttachmentIsNotTheBody(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=B\r\n\r\n"+ + "--B\r\nContent-Type: text/plain\r\nContent-Disposition: attachment; filename=log.txt\r\n\r\nattached log\r\n"+ + "--B\r\nContent-Type: text/plain\r\n\r\nthe real message\r\n--B--\r\n") + if strings.TrimSpace(e.Body) != "the real message" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestFirstTextPartWins(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=B\r\n\r\n"+ + "--B\r\nContent-Type: text/plain\r\n\r\nmessage\r\n"+ + "--B\r\nContent-Type: text/plain\r\n\r\n-- \r\nsignature footer\r\n--B--\r\n") + if strings.TrimSpace(e.Body) != "message" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestCapNeverSplitsAMultibyteCharacter(t *testing.T) { + // 3-byte characters: MaxBody is not a multiple of 3, so a blind cut lands mid-character. + e := parse(t, "Subject: big\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+strings.Repeat("€", MaxBody/3+10)+"\r\n") + if !utf8.ValidString(e.Body) || strings.ContainsRune(e.Body, utf8.RuneError) { + t.Fatal("body was cut in the middle of a character") + } + if len(e.Body) > MaxBody || len(e.Body) < MaxBody-3 { + t.Fatalf("body len %d", len(e.Body)) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 83027b5..dd33a8b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,6 +20,9 @@ type Config struct { API struct { Listen string `yaml:"listen"` // HTTP listen address; the bearer token comes from PARSIMONY_API_TOKEN } `yaml:"api"` + Web struct { + Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them + } `yaml:"web"` Relay struct { Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay } `yaml:"relay"` @@ -33,6 +36,7 @@ func Load(path string) (Config, error) { c.LMTP.Addr = "127.0.0.1:2424" c.Relay.Addr = "127.0.0.1:25" c.API.Listen = "127.0.0.1:8080" + c.Web.Listen = "127.0.0.1:8081" b, err := os.ReadFile(path) if err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 99d0729..0e39f69 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -20,17 +20,17 @@ func TestLoadDefaults(t *testing.T) { if err != nil { t.Fatal(err) } - if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.API.Listen != "127.0.0.1:8080" { + if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" { t.Fatalf("defaults: %+v", c) } } func TestLoadValues(t *testing.T) { - c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\napi:\n listen: 0.0.0.0:9000\n")) + c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n")) if err != nil { t.Fatal(err) } - if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.API.Listen != "0.0.0.0:9000" { + if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" { t.Fatalf("values: %+v", c) } } @@ -46,3 +46,13 @@ func TestLoadUnknownField(t *testing.T) { t.Fatal("unknown fields must be rejected") } } + +func TestWebCanBeDisabled(t *testing.T) { + c, err := Load(write(t, "web:\n listen: \"\"\n")) + if err != nil { + t.Fatal(err) + } + if c.Web.Listen != "" { + t.Fatalf("an empty listen address disables the web: %q", c.Web.Listen) + } +} diff --git a/internal/router/m3_test.go b/internal/router/m3_test.go index 3a70c98..6769095 100644 --- a/internal/router/m3_test.go +++ b/internal/router/m3_test.go @@ -201,3 +201,14 @@ func TestBounceAndUnsubscribeTokensAreNotInterchangeable(t *testing.T) { t.Fatalf("a bounce MAC must not unsubscribe: %v", err) } } + +func TestUnsubscribeRejectsMemberIDZero(t *testing.T) { + e := newEnv(t) + tok := fmt.Sprintf("0-%s", e.r.mac("unsub", 0)) // correctly signed, but not a real member id + if err := e.r.Unsubscribe(tok); !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } + if _, err := e.r.CheckUnsubscribe(tok); !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } +} diff --git a/internal/web/web_test.go b/internal/web/web_test.go index ba41751..bbdd4c4 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -152,6 +152,9 @@ func TestArchivePaging(t *testing.T) { wantStatus(t, p2, 200) wantBody(t, p2, "Subject number 0<") wantNoBody(t, p2, "page=3") + // A page past the end shows the last page instead of an empty one. + far := get(t, f.h, "/archive/dev@example.org?page=999") + wantBody(t, far, "Subject number 0<") for _, bad := range []string{"?page=abc", "?page=0", "?page=-4", "?page=999"} { wantStatus(t, get(t, f.h, "/archive/dev@example.org"+bad), 200) } diff --git a/parsimony.example.yaml b/parsimony.example.yaml index 7240253..baea5be 100644 --- a/parsimony.example.yaml +++ b/parsimony.example.yaml @@ -5,6 +5,8 @@ lmtp: addr: 0.0.0.0:2424 relay: addr: postfix:25 +web: + listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them api: listen: 0.0.0.0:8080 # The API token is read from the PARSIMONY_API_TOKEN environment variable.