diff --git a/README.md b/README.md index 042171c..f3ab39e 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,10 @@ A modern, single-binary mailing list manager: an evolution of Mailman. - Postfix delivers to parsimony over LMTP; parsimony relays through SMTP. - Discussion lists with member-only posting, `List-*` headers, one-click unsubscribe and DMARC-safe `From` rewriting. +- Self-service subscribe and unsubscribe by email with double opt-in: + send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. +- Moderated lists: posts wait in a queue that the API can approve or discard. +- VERP bounces: every recipient gets a signed return address; 3 hard bounces disable a member. - SQLite storage, one YAML config file. - Admin REST API under `/api/v1` (see `docs/openapi.yaml`). @@ -20,3 +24,13 @@ The API is disabled when `PARSIMONY_API_TOKEN` is not set. ``` curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists ``` + +## Postfix + +Deliver the list domains to parsimony over LMTP and keep the `+` extensions intact, for example: + +``` +transport_maps = hash:/etc/postfix/transport +# /etc/postfix/transport +lists.example.org lmtp:inet:127.0.0.1:2424 +``` diff --git a/docs/openapi.yaml b/docs/openapi.yaml index ad0a4db..16c2b82 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -20,11 +20,23 @@ components: domain: {type: string} title: {type: string} address: {type: string} + post_policy: {type: string, enum: [members, moderated]} + join_policy: {type: string, enum: [open, closed]} Member: type: object properties: address: {type: string} display_name: {type: string} + disabled: {type: boolean, description: Disabled after 3 hard bounces; re-adding the member re-enables it} + bounce_count: {type: integer} + Held: + type: object + properties: + id: {type: integer} + sender: {type: string} + subject: {type: string} + size: {type: integer} + created_at: {type: string, format: date-time} Error: type: object properties: @@ -85,16 +97,17 @@ paths: schema: {$ref: '#/components/schemas/List'} '404': {$ref: '#/components/responses/Error'} patch: - summary: Update the list title + summary: Update title and policies (send only the fields to change) requestBody: required: true content: application/json: schema: type: object - required: [title] properties: title: {type: string} + post_policy: {type: string, enum: [members, moderated]} + join_policy: {type: string, enum: [open, closed]} responses: '200': description: OK @@ -149,3 +162,32 @@ paths: responses: '204': {description: Removed} '404': {$ref: '#/components/responses/Error'} + /lists/{id}/held: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + get: + summary: Messages waiting for moderation + responses: + '200': + description: OK + content: + application/json: + schema: {type: array, items: {$ref: '#/components/schemas/Held'}} + '404': {$ref: '#/components/responses/Error'} + /held/{id}/approve: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + post: + summary: Approve and deliver a held message + responses: + '204': {description: Delivered} + '404': {$ref: '#/components/responses/Error'} + '501': {$ref: '#/components/responses/Error'} + /held/{id}: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + delete: + summary: Discard a held message + responses: + '204': {description: Discarded} + '404': {$ref: '#/components/responses/Error'} diff --git a/internal/api/api.go b/internal/api/api.go index 9dbb9d8..1d027bf 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -2,26 +2,37 @@ package api import ( + "bytes" + "context" "crypto/subtle" "encoding/json" "errors" + "mime" "net/http" "net/mail" "strconv" "strings" + "time" "codeberg.org/squidd/parsimony/internal/store" ) +// Releaser approves a held message: it delivers it to the list. +type Releaser interface { + Release(ctx context.Context, heldID int64) error +} + type server struct { - store *store.Store - token string + store *store.Store + token string + releaser Releaser } // New returns the API handler. Every route except /api/v1/health needs // "Authorization: Bearer ". An empty token rejects all requests. -func New(s *store.Store, token string) http.Handler { - srv := &server{store: s, token: token} +// rel may be nil: approving held messages then answers 501. +func New(s *store.Store, token string, rel Releaser) http.Handler { + srv := &server{store: s, token: token, releaser: rel} mux := http.NewServeMux() mux.HandleFunc("GET /api/v1/health", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) @@ -34,6 +45,9 @@ func New(s *store.Store, token string) http.Handler { mux.Handle("GET /api/v1/lists/{id}/members", srv.auth(srv.listMembers)) mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember)) mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember)) + mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld)) + mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld)) + mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld)) return mux } @@ -57,15 +71,21 @@ type listDTO struct { Domain string `json:"domain"` Title string `json:"title"` Address string `json:"address"` + + PostPolicy string `json:"post_policy"` + JoinPolicy string `json:"join_policy"` } func toListDTO(l store.List) listDTO { - return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address()} + return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address(), + PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy} } type memberDTO struct { Address string `json:"address"` DisplayName string `json:"display_name"` + Disabled bool `json:"disabled"` + BounceCount int `json:"bounce_count"` } // Helpers. @@ -138,8 +158,8 @@ func (s *server) createList(w http.ResponseWriter, r *http.Request) { return } in.Name, in.Domain = strings.TrimSpace(in.Name), strings.TrimSpace(in.Domain) - if in.Name == "" || in.Domain == "" || strings.ContainsAny(in.Name+in.Domain, "@ \r\n") { - writeError(w, http.StatusBadRequest, "name and domain are required and must not contain '@' or spaces") + if in.Name == "" || in.Domain == "" || strings.ContainsAny(in.Name+in.Domain, "@+ \r\n") { + writeError(w, http.StatusBadRequest, "name and domain are required and must not contain '@', '+' or spaces") return } if in.Title == "" { @@ -172,18 +192,15 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { return } var in struct { - Title string `json:"title"` + Title *string `json:"title"` + PostPolicy *string `json:"post_policy"` + JoinPolicy *string `json:"join_policy"` } if !decodeBody(w, r, &in) { return } - title := strings.TrimSpace(in.Title) - if title == "" || strings.ContainsAny(title, "\r\n") { - writeError(w, http.StatusBadRequest, "title is required and must be a single line") - return - } - if err := s.store.UpdateList(id, title); err != nil { - storeError(w, err) + if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil { + writeError(w, http.StatusBadRequest, "nothing to update") return } l, err := s.store.GetList(id) @@ -191,6 +208,34 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { storeError(w, err) return } + if in.Title != nil { + title := strings.TrimSpace(*in.Title) + if title == "" || strings.ContainsAny(title, "\r\n") { + writeError(w, http.StatusBadRequest, "title must be a non-empty single line") + return + } + l.Title = title + } + if in.PostPolicy != nil { + l.PostPolicy = *in.PostPolicy + } + if in.JoinPolicy != nil { + l.JoinPolicy = *in.JoinPolicy + } + // Validate before writing anything. + if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated) || + (l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) { + writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy must be open or closed") + return + } + if err := s.store.UpdateList(id, l.Title); err != nil { + storeError(w, err) + return + } + if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy); err != nil { + storeError(w, err) + return + } writeJSON(w, http.StatusOK, toListDTO(l)) } @@ -224,7 +269,7 @@ func (s *server) listMembers(w http.ResponseWriter, r *http.Request) { } out := make([]memberDTO, 0, len(ms)) for _, m := range ms { - out = append(out, memberDTO{Address: m.Address, DisplayName: m.DisplayName}) + out = append(out, memberDTO{Address: m.Address, DisplayName: m.DisplayName, Disabled: m.Disabled, BounceCount: m.BounceCount}) } writeJSON(w, http.StatusOK, out) } @@ -269,3 +314,74 @@ func (s *server) removeMember(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(http.StatusNoContent) } + +// Handlers: held messages. + +type heldDTO struct { + ID int64 `json:"id"` + Sender string `json:"sender"` + Subject string `json:"subject"` + Size int `json:"size"` + CreatedAt time.Time `json:"created_at"` +} + +func subjectOf(data []byte) string { + m, err := mail.ReadMessage(bytes.NewReader(data)) + if err != nil { + return "" + } + subj := m.Header.Get("Subject") + if dec, err := new(mime.WordDecoder).DecodeHeader(subj); err == nil { + return dec + } + return subj +} + +func (s *server) listHeld(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + if _, err := s.store.GetList(id); err != nil { + storeError(w, err) + return + } + hs, err := s.store.ListHeld(id) + if err != nil { + storeError(w, err) + return + } + out := make([]heldDTO, 0, len(hs)) + for _, h := range hs { + out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: subjectOf(h.Data), Size: len(h.Data), CreatedAt: h.CreatedAt}) + } + writeJSON(w, http.StatusOK, out) +} + +func (s *server) approveHeld(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + if s.releaser == nil { + writeError(w, http.StatusNotImplemented, "delivery is not configured") + return + } + if err := s.releaser.Release(r.Context(), id); err != nil { + storeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) discardHeld(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + if err := s.store.DeleteHeld(id); err != nil { + storeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 5d583e4..92382a7 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "net/http" "net/http/httptest" @@ -19,7 +20,7 @@ func newServer(t *testing.T) http.Handler { t.Fatal(err) } t.Cleanup(func() { s.Close() }) - return New(s, token) + return New(s, token, nil) } func do(t *testing.T, h http.Handler, method, path, body string, auth bool) *httptest.ResponseRecorder { @@ -68,7 +69,7 @@ func TestAuthRequired(t *testing.T) { func TestEmptyTokenRejectsEverything(t *testing.T) { s, _ := store.Open(":memory:") defer s.Close() - h := New(s, "") + h := New(s, "", nil) req := httptest.NewRequest("GET", "/api/v1/lists", nil) req.Header.Set("Authorization", "Bearer ") rec := httptest.NewRecorder() @@ -77,11 +78,13 @@ func TestEmptyTokenRejectsEverything(t *testing.T) { } type listJSON struct { - ID int64 `json:"id"` - Name string `json:"name"` - Domain string `json:"domain"` - Title string `json:"title"` - Address string `json:"address"` + ID int64 `json:"id"` + Name string `json:"name"` + Domain string `json:"domain"` + Title string `json:"title"` + Address string `json:"address"` + PostPolicy string `json:"post_policy"` + JoinPolicy string `json:"join_policy"` } func createList(t *testing.T, h http.Handler) listJSON { @@ -141,6 +144,7 @@ func TestCreateListValidation(t *testing.T) { `{}`, `{"name":"","domain":"example.org"}`, `{"name":"a@b","domain":"example.org"}`, + `{"name":"a+b","domain":"example.org"}`, `{"name":"dev","domain":""}`, `{"name":"dev","domain":"example.org","bogus":1}`, } { @@ -159,6 +163,8 @@ func TestCreateListValidation(t *testing.T) { type memberJSON struct { Address string `json:"address"` DisplayName string `json:"display_name"` + Disabled bool `json:"disabled"` + BounceCount int `json:"bounce_count"` } func TestMembers(t *testing.T) { @@ -194,3 +200,118 @@ func TestMembers(t *testing.T) { expect(t, do(t, h, "DELETE", "/api/v1/lists/1/members/alice%40example.com", "", true), 204) expect(t, do(t, h, "DELETE", "/api/v1/lists/1/members/alice%40example.com", "", true), 404) } + +func TestListPoliciesViaAPI(t *testing.T) { + h := newServer(t) + l := createList(t, h) + if l.PostPolicy != "members" || l.JoinPolicy != "open" { + t.Fatalf("defaults: %+v", l) + } + rec := do(t, h, "PATCH", "/api/v1/lists/1", `{"post_policy":"moderated"}`, true) + expect(t, rec, 200) + var u listJSON + decode(t, rec, &u) + if u.PostPolicy != "moderated" || u.JoinPolicy != "open" || u.Title != "Dev" { + t.Fatalf("patched: %+v", u) + } + expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"join_policy":"closed","title":"New"}`, true), 200) + rec = do(t, h, "GET", "/api/v1/lists/1", "", true) + decode(t, rec, &u) + if u.JoinPolicy != "closed" || u.Title != "New" || u.PostPolicy != "moderated" { + t.Fatalf("after: %+v", u) + } + expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"post_policy":"bogus"}`, true), 400) +} + +func TestMemberBounceState(t *testing.T) { + s, _ := store.Open(":memory:") + defer s.Close() + h := New(s, token, nil) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "a@x.com", "A") + ms, _ := s.Members(l.ID) + for i := 0; i < 3; i++ { + s.RecordHardBounce(ms[0].ID, 3) + } + rec := do(t, h, "GET", "/api/v1/lists/1/members", "", true) + var out []memberJSON + decode(t, rec, &out) + if len(out) != 1 || !out[0].Disabled || out[0].BounceCount != 3 { + t.Fatalf("members: %+v", out) + } + // Re-adding re-enables. + expect(t, do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com"}`, true), 201) + rec = do(t, h, "GET", "/api/v1/lists/1/members", "", true) + decode(t, rec, &out) + if out[0].Disabled || out[0].BounceCount != 0 { + t.Fatalf("after re-add: %+v", out) + } +} + +type fakeReleaser struct{ released []int64 } + +func (f *fakeReleaser) Release(_ context.Context, id int64) error { + f.released = append(f.released, id) + return nil +} + +func heldServer(t *testing.T) (http.Handler, *store.Store, *fakeReleaser) { + t.Helper() + s, _ := store.Open(":memory:") + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.HoldMessage(l.ID, "Bob@x.com", []byte("From: Bob \r\nSubject: Hello there\r\n\r\nbody")) + rel := &fakeReleaser{} + return New(s, token, rel), s, rel +} + +type heldJSON struct { + ID int64 `json:"id"` + Sender string `json:"sender"` + Subject string `json:"subject"` + Size int `json:"size"` +} + +func TestHeldQueueAPI(t *testing.T) { + h, _, rel := heldServer(t) + rec := do(t, h, "GET", "/api/v1/lists/1/held", "", true) + expect(t, rec, 200) + var hs []heldJSON + decode(t, rec, &hs) + if len(hs) != 1 || hs[0].Sender != "bob@x.com" || hs[0].Subject != "Hello there" || hs[0].Size == 0 { + t.Fatalf("held: %+v", hs) + } + expect(t, do(t, h, "GET", "/api/v1/lists/99/held", "", true), 404) + + expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", true), 204) + if len(rel.released) != 1 || rel.released[0] != 1 { + t.Fatalf("released: %v", rel.released) + } + expect(t, do(t, h, "DELETE", "/api/v1/held/1", "", true), 204) + expect(t, do(t, h, "DELETE", "/api/v1/held/1", "", true), 404) + expect(t, do(t, h, "DELETE", "/api/v1/held/abc", "", true), 404) +} + +func TestHeldRequiresAuth(t *testing.T) { + h, _, _ := heldServer(t) + expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", false), 401) + expect(t, do(t, h, "GET", "/api/v1/lists/1/held", "", false), 401) +} + +func TestApproveWithoutReleaser(t *testing.T) { + s, _ := store.Open(":memory:") + defer s.Close() + h := New(s, token, nil) + expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", true), 501) +} + +func TestApproveUnknownHeld(t *testing.T) { + s, _ := store.Open(":memory:") + defer s.Close() + h := New(s, token, errReleaser{store.ErrNotFound}) + expect(t, do(t, h, "POST", "/api/v1/held/5/approve", "", true), 404) +} + +type errReleaser struct{ err error } + +func (e errReleaser) Release(context.Context, int64) error { return e.err } diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 4ecf745..3fa60c0 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -77,22 +77,33 @@ func (s *session) LMTPData(r io.Reader, status smtp.StatusCollector) error { } func (s *session) deliver(ctx context.Context, rcpt string, data []byte) error { - res, err := s.b.router.Route(s.from, rcpt, data) + rt := s.b.router + if handled, err := rt.Command(ctx, s.b.out, s.from, rcpt, data); handled { + return s.status(rcpt, err) + } + res, err := rt.Route(s.from, rcpt, data) + if err != nil { + return s.status(rcpt, err) + } + return s.status(rcpt, rt.Send(ctx, s.b.out, res)) +} + +// status turns a router result into an LMTP reply. +func (s *session) status(rcpt string, err error) error { switch { + case err == nil, errors.Is(err, router.ErrHeld): + return nil case errors.Is(err, router.ErrUnknownList): return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such list"} case errors.Is(err, router.ErrNotMember): return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "only list members may post"} - case err != nil: - log.Printf("route %s: %v", rcpt, err) - return tempFail() + case errors.Is(err, router.ErrBadToken): + return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "invalid or expired token"} + case errors.Is(err, router.ErrJoinClosed): + return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "this list does not accept subscriptions"} } - bounce := res.List.Name + "+bounce@" + res.List.Domain - if err := s.b.out.Send(ctx, bounce, res.Recipients, res.Data); err != nil { - log.Printf("relay %s: %v", rcpt, err) - return tempFail() - } - return nil + log.Printf("deliver %s: %v", rcpt, err) + return tempFail() } func tempFail() error { diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index e76d426..051c042 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -85,14 +85,14 @@ func TestDeliverFansOut(t *testing.T) { if res["dev@example.org"] != nil { t.Fatalf("unexpected error: %v", res) } - if len(relay.Sent) != 1 { - t.Fatalf("want 1 send, got %d", len(relay.Sent)) + if len(relay.Sent) != 2 { + t.Fatalf("want one send per member, got %d", len(relay.Sent)) } s := relay.Sent[0] - if len(s.To) != 2 { + if len(s.To) != 1 { t.Errorf("recipients = %v", s.To) } - if s.From != "dev+bounce@example.org" { + if !strings.HasPrefix(s.From, "dev+bounce-") { t.Errorf("envelope from = %q", s.From) } if !strings.Contains(string(s.Data), "List-Id: Dev ") { @@ -127,7 +127,7 @@ func TestMixedRecipients(t *testing.T) { if res["dev@example.org"] != nil || res["nope@example.org"] == nil { t.Fatalf("results: %v", res) } - if len(relay.Sent) != 1 { + if len(relay.Sent) != 2 { t.Fatalf("sent = %d", len(relay.Sent)) } } @@ -154,3 +154,47 @@ func TestRelayFailureIsTemporary(t *testing.T) { t.Fatalf("want 451, got %v", res["dev@example.org"]) } } + +func TestJoinCommandOverLMTP(t *testing.T) { + addr, relay := setup(t) + res := deliver(t, addr, "carol@x.com", []string{"dev+join@example.org"}) + if res["dev+join@example.org"] != nil { + t.Fatalf("join: %v", res) + } + if len(relay.Sent) != 1 || relay.Sent[0].To[0] != "carol@x.com" { + t.Fatalf("confirmation mail: %+v", relay.Sent) + } +} + +func TestBadTokenRejected(t *testing.T) { + addr, _ := setup(t) + res := deliver(t, addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"}) + var se *smtp.SMTPError + if !errors.As(res["dev+confirm-deadbeef@example.org"], &se) || se.Code != 550 { + t.Fatalf("want 550, got %v", res) + } +} + +func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) { + s, _ := store.Open(":memory:") + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "alice@example.com", "Alice") + s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen) + relay := &outbound.Fake{} + srv := NewServer(router.New(s), relay) + ln, _ := net.Listen("tcp", "127.0.0.1:0") + go srv.Serve(ln) + t.Cleanup(func() { srv.Close() }) + + res := deliver(t, ln.Addr().String(), "stranger@x.com", []string{"dev@example.org"}) + if res["dev@example.org"] != nil { + t.Fatalf("held mail must be accepted: %v", res) + } + if len(relay.Sent) != 0 { + t.Fatal("nothing may be relayed") + } + if h, _ := s.ListHeld(l.ID); len(h) != 1 { + t.Fatalf("held = %d", len(h)) + } +} diff --git a/internal/router/commands.go b/internal/router/commands.go new file mode 100644 index 0000000..97d852d --- /dev/null +++ b/internal/router/commands.go @@ -0,0 +1,174 @@ +package router + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "mime" + "net/mail" + "regexp" + "strconv" + "strings" + "time" + + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/store" +) + +// bounceThreshold is the number of hard bounces after which a member is disabled. +const bounceThreshold = 3 + +var statusRe = regexp.MustCompile(`(?mi)^Status:\s*([245])\.\d+\.\d+`) + +func (r *Router) secret() ([]byte, error) { + r.secretOnce.Do(func() { r.secretVal, r.secretErr = r.store.Secret() }) + return r.secretVal, r.secretErr +} + +func (r *Router) mac(memberID int64) string { + key, err := r.secret() + if err != nil { + return "" + } + h := hmac.New(sha256.New, key) + fmt.Fprintf(h, "bounce:%d", memberID) + return hex.EncodeToString(h.Sum(nil))[:12] +} + +// bounceAddress returns the VERP envelope sender for a member, or "" on error. +func (r *Router) bounceAddress(l store.List, memberID int64) string { + mac := r.mac(memberID) + if mac == "" { + return "" + } + return fmt.Sprintf("%s+bounce-%d-%s@%s", l.Name, memberID, mac, l.Domain) +} + +// Command handles list command addresses such as dev+join@example.org. +// handled is false when rcpt is a plain list address (use Route for those). +func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt string, data []byte) (handled bool, err error) { + local, domain, ok := strings.Cut(strings.ToLower(strings.TrimSpace(rcpt)), "@") + if !ok { + return false, nil + } + name, tag, hasTag := strings.Cut(local, "+") + if !hasTag { + return false, nil + } + l, err := r.store.FindList(name + "@" + domain) + if errors.Is(err, store.ErrNotFound) { + return true, ErrUnknownList + } + if err != nil { + return true, err + } + + switch { + case tag == "join": + return true, r.join(ctx, out, l, sender) + case tag == "leave": + return true, r.leave(ctx, out, l, sender) + case strings.HasPrefix(tag, "confirm-"): + return true, r.confirm(l, strings.TrimPrefix(tag, "confirm-")) + case tag == "bounce": + return true, nil // system mail without a token: nothing to learn from it + case strings.HasPrefix(tag, "bounce-"): + return true, r.bounce(l, strings.TrimPrefix(tag, "bounce-"), data) + } + return true, ErrUnknownList +} + +func (r *Router) join(ctx context.Context, out outbound.Sender, l store.List, sender string) error { + if sender == "" { + return nil + } + if l.JoinPolicy != store.JoinOpen { + return ErrJoinClosed + } + if ok, err := r.store.IsMember(l.ID, sender); err != nil || ok { + return err // members get no mail: it would leak membership + } + return r.askConfirmation(ctx, out, l, sender, store.ActionJoin) +} + +func (r *Router) leave(ctx context.Context, out outbound.Sender, l store.List, sender string) error { + if sender == "" { + return nil + } + if ok, err := r.store.IsMember(l.ID, sender); err != nil || !ok { + return err + } + return r.askConfirmation(ctx, out, l, sender, store.ActionLeave) +} + +func (r *Router) askConfirmation(ctx context.Context, out outbound.Sender, l store.List, addr, action string) error { + token, created, err := r.store.CreatePending(l.ID, addr, action) + if err != nil || !created { + return err + } + verb, subject := "join", "Confirm your subscription to "+l.Title + if action == store.ActionLeave { + verb, subject = "leave", "Confirm your unsubscription from "+l.Title + } + msg := fmt.Sprintf("From: %s\r\nTo: %s\r\nReply-To: %s+confirm-%s@%s\r\nSubject: %s\r\nDate: %s\r\n"+ + "Message-ID: <%s.%s@%s>\r\nAuto-Submitted: auto-generated\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ + "Someone, probably you, asked to %s %s.\r\n\r\n"+ + "To confirm, just reply to this message. The request expires in 72 hours.\r\n"+ + "If you did not ask for this, ignore this message and nothing will happen.\r\n", + (&mail.Address{Name: l.Title, Address: l.Address()}).String(), + (&mail.Address{Address: addr}).String(), + l.Name, token, l.Domain, + mime.QEncoding.Encode("utf-8", subject), + time.Now().Format(time.RFC1123Z), + token[:8], strconv.FormatInt(time.Now().UnixNano(), 36), l.Domain, + verb, l.Address()) + return out.Send(ctx, l.Name+"+bounce@"+l.Domain, []string{addr}, []byte(msg)) +} + +func (r *Router) confirm(l store.List, token string) error { + p, err := r.store.ConfirmPending(token) + if errors.Is(err, store.ErrNotFound) || (err == nil && p.ListID != l.ID) { + return ErrBadToken + } + if err != nil { + return err + } + switch p.Action { + case store.ActionJoin: + if l.JoinPolicy != store.JoinOpen { + return ErrJoinClosed + } + return r.store.AddMember(l.ID, p.Address, "") + case store.ActionLeave: + if err := r.store.RemoveMember(l.ID, p.Address); err != nil && !errors.Is(err, store.ErrNotFound) { + return err + } + } + return nil +} + +func (r *Router) bounce(l store.List, tag string, data []byte) error { + idStr, mac, ok := strings.Cut(tag, "-") + id, err := strconv.ParseInt(idStr, 10, 64) + if !ok || err != nil { + return ErrBadToken + } + want := r.mac(id) + if want == "" || !hmac.Equal([]byte(mac), []byte(want)) { + return ErrBadToken + } + m, err := r.store.MemberByID(id) + if errors.Is(err, store.ErrNotFound) || (err == nil && m.ListID != l.ID) { + return ErrBadToken + } + if err != nil { + return err + } + if sm := statusRe.FindSubmatch(data); sm != nil && string(sm[1]) == "5" { + _, err = r.store.RecordHardBounce(m.ID, bounceThreshold) + } + return err +} diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go new file mode 100644 index 0000000..8cde385 --- /dev/null +++ b/internal/router/m2_test.go @@ -0,0 +1,316 @@ +package router + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/store" +) + +type env struct { + r *Router + s *store.Store + out *outbound.Fake + list store.List + ctx context.Context + alice int64 +} + +func newEnv(t *testing.T) *env { + t.Helper() + s, err := store.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "alice@example.com", "Alice") + s.AddMember(l.ID, "bob@example.com", "Bob") + ms, _ := s.Members(l.ID) + return &env{r: New(s), s: s, out: &outbound.Fake{}, list: l, ctx: context.Background(), alice: ms[0].ID} +} + +func lastMail(t *testing.T, out *outbound.Fake) outbound.Sent { + t.Helper() + if len(out.Sent) == 0 { + t.Fatal("no mail was sent") + } + return out.Sent[len(out.Sent)-1] +} + +func TestSendUsesVERPPerRecipient(t *testing.T) { + e := newEnv(t) + res, err := e.r.Route("alice@example.com", "dev@example.org", []byte(sample)) + if err != nil { + t.Fatal(err) + } + if err := e.r.Send(e.ctx, e.out, res); err != nil { + t.Fatal(err) + } + if len(e.out.Sent) != 2 { + t.Fatalf("want one send per recipient, got %d", len(e.out.Sent)) + } + seen := map[string]bool{} + for _, m := range e.out.Sent { + if len(m.To) != 1 { + t.Fatalf("to = %v", m.To) + } + if !strings.HasPrefix(m.From, "dev+bounce-") || !strings.HasSuffix(m.From, "@example.org") { + t.Fatalf("envelope from = %q", m.From) + } + seen[m.From] = true + } + if len(seen) != 2 { + t.Fatal("each recipient needs a distinct bounce address") + } +} + +func TestDisabledMembersAreSkipped(t *testing.T) { + e := newEnv(t) + for i := 0; i < 3; i++ { + e.s.RecordHardBounce(e.alice, 3) + } + res, err := e.r.Route("bob@example.com", "dev@example.org", []byte(sample)) + if err != nil { + t.Fatal(err) + } + if len(res.Recipients) != 1 || res.Recipients[0].Address != "bob@example.com" { + t.Fatalf("recipients: %+v", res.Recipients) + } +} + +func TestModeratedListHoldsPosts(t *testing.T) { + e := newEnv(t) + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen) + for _, sender := range []string{"alice@example.com", "stranger@x.com"} { + _, err := e.r.Route(sender, "dev@example.org", []byte(sample)) + if !errors.Is(err, ErrHeld) { + t.Fatalf("%s: want ErrHeld, got %v", sender, err) + } + } + held, _ := e.s.ListHeld(e.list.ID) + if len(held) != 2 { + t.Fatalf("held: %d", len(held)) + } + if len(e.out.Sent) != 0 { + t.Fatal("nothing may be sent while held") + } +} + +func TestReleaseHeld(t *testing.T) { + e := newEnv(t) + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen) + e.r.Route("stranger@x.com", "dev@example.org", []byte(sample)) + held, _ := e.s.ListHeld(e.list.ID) + + if err := e.r.Release(e.ctx, e.out, held[0].ID); err != nil { + t.Fatal(err) + } + if len(e.out.Sent) != 2 { + t.Fatalf("sent = %d", len(e.out.Sent)) + } + if !strings.Contains(string(e.out.Sent[0].Data), "List-Id: Dev ") { + t.Error("released mail must carry list headers") + } + if rest, _ := e.s.ListHeld(e.list.ID); len(rest) != 0 { + t.Fatal("released message must leave the queue") + } + if err := e.r.Release(e.ctx, e.out, held[0].ID); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func token(t *testing.T, m outbound.Sent) string { + t.Helper() + const marker = "Reply-To: dev+confirm-" + i := strings.Index(string(m.Data), marker) + if i < 0 { + t.Fatalf("no confirm address in %q", m.Data) + } + rest := string(m.Data)[i+len(marker):] + return rest[:strings.Index(rest, "@")] +} + +func TestJoinFlow(t *testing.T) { + e := newEnv(t) + handled, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + if !handled || err != nil { + t.Fatalf("join: %v %v", handled, err) + } + m := lastMail(t, e.out) + if len(m.To) != 1 || m.To[0] != "carol@x.com" { + t.Fatalf("confirmation must go to the requester: %+v", m.To) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok { + t.Fatal("not a member before confirming") + } + tok := token(t, m) + + handled, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) + if !handled || err != nil { + t.Fatalf("confirm: %v %v", handled, err) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok { + t.Fatal("carol must be a member after confirming") + } + _, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) + if !errors.Is(err, ErrBadToken) { + t.Fatalf("reused token: want ErrBadToken, got %v", err) + } +} + +func TestJoinRateLimitedAndSilentForMembers(t *testing.T) { + e := newEnv(t) + e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + if len(e.out.Sent) != 1 { + t.Fatalf("second request must not send mail, sent %d", len(e.out.Sent)) + } + e.r.Command(e.ctx, e.out, "alice@example.com", "dev+join@example.org", nil) + if len(e.out.Sent) != 1 { + t.Fatal("existing members get no mail") + } + e.r.Command(e.ctx, e.out, "", "dev+join@example.org", nil) + if len(e.out.Sent) != 1 { + t.Fatal("empty sender is ignored") + } +} + +func TestJoinClosed(t *testing.T) { + e := newEnv(t) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed) + _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + if !errors.Is(err, ErrJoinClosed) { + t.Fatalf("want ErrJoinClosed, got %v", err) + } + if len(e.out.Sent) != 0 { + t.Fatal("no mail for closed lists") + } +} + +func TestConfirmAfterListClosed(t *testing.T) { + e := newEnv(t) + e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + tok := token(t, lastMail(t, e.out)) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed) + _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) + if !errors.Is(err, ErrJoinClosed) { + t.Fatalf("want ErrJoinClosed, got %v", err) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok { + t.Fatal("must not join a closed list") + } +} + +func TestLeaveFlow(t *testing.T) { + e := newEnv(t) + e.r.Command(e.ctx, e.out, "alice@example.com", "dev+leave@example.org", nil) + m := lastMail(t, e.out) + if m.To[0] != "alice@example.com" { + t.Fatalf("to = %v", m.To) + } + if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); !ok { + t.Fatal("still a member before confirming") + } + e.r.Command(e.ctx, e.out, "alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil) + if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); ok { + t.Fatal("alice must be gone") + } + // Leaving when not a member sends nothing. + n := len(e.out.Sent) + e.r.Command(e.ctx, e.out, "zed@x.com", "dev+leave@example.org", nil) + if len(e.out.Sent) != n { + t.Fatal("non-members get no mail") + } +} + +func TestConfirmTokenFromOtherList(t *testing.T) { + e := newEnv(t) + e.s.CreateList("ops", "example.org", "Ops") + e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + tok := token(t, lastMail(t, e.out)) + _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "ops+confirm-"+tok+"@example.org", nil) + if !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } +} + +func TestCommandNotHandledForPlainAddress(t *testing.T) { + e := newEnv(t) + handled, err := e.r.Command(e.ctx, e.out, "a@x.com", "dev@example.org", nil) + if handled || err != nil { + t.Fatalf("plain address: %v %v", handled, err) + } +} + +func TestUnknownCommandAndList(t *testing.T) { + e := newEnv(t) + for _, rcpt := range []string{"dev+bogus@example.org", "nope+join@example.org"} { + handled, err := e.r.Command(e.ctx, e.out, "a@x.com", rcpt, nil) + if !handled || !errors.Is(err, ErrUnknownList) { + t.Errorf("%s: %v %v", rcpt, handled, err) + } + } +} + +func bounceMsg(status string) []byte { + return []byte(fmt.Sprintf("Subject: Undelivered Mail\r\nContent-Type: text/plain\r\n\r\nAction: failed\r\nStatus: %s\r\nDiagnostic-Code: smtp; 550 no such user\r\n", status)) +} + +func TestHardBouncesDisableMember(t *testing.T) { + e := newEnv(t) + addr := e.r.bounceAddress(e.list, e.alice) + for i := 0; i < 3; i++ { + handled, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1")) + if !handled || err != nil { + t.Fatalf("bounce: %v %v", handled, err) + } + } + m, _ := e.s.MemberByID(e.alice) + if !m.Disabled || m.BounceCount != 3 { + t.Fatalf("member: %+v", m) + } +} + +func TestSoftBounceIgnored(t *testing.T) { + e := newEnv(t) + e.r.Command(e.ctx, e.out, "", e.r.bounceAddress(e.list, e.alice), bounceMsg("4.2.2")) + if m, _ := e.s.MemberByID(e.alice); m.BounceCount != 0 { + t.Fatalf("soft bounce counted: %+v", m) + } +} + +func TestForgedBounceRejected(t *testing.T) { + e := newEnv(t) + addr := fmt.Sprintf("dev+bounce-%d-000000000000@example.org", e.alice) + _, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1")) + if !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } + if m, _ := e.s.MemberByID(e.alice); m.BounceCount != 0 { + t.Fatal("forged bounce must not count") + } +} + +func TestBounceFromOtherListRejected(t *testing.T) { + e := newEnv(t) + e.s.CreateList("ops", "example.org", "Ops") + ops, _ := e.s.FindList("ops@example.org") + addr := e.r.bounceAddress(ops, e.alice) // valid MAC but alice is not in ops + _, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1")) + if !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } +} + +func TestSystemBounceAddressDiscarded(t *testing.T) { + e := newEnv(t) + handled, err := e.r.Command(e.ctx, e.out, "", "dev+bounce@example.org", bounceMsg("5.1.1")) + if !handled || err != nil { + t.Fatalf("%v %v", handled, err) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 4e40206..0bf7d91 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -2,23 +2,40 @@ package router import ( + "context" "errors" + "fmt" + "log" + "sync" + "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/store" ) var ( - // ErrUnknownList means the recipient address is not a list. + // ErrUnknownList means the recipient address is not a list or a list command. ErrUnknownList = errors.New("unknown list") // ErrNotMember means the sender may not post to the list. ErrNotMember = errors.New("sender is not a member") + // ErrHeld means the message was queued for moderation. It is not a failure. + ErrHeld = errors.New("held for moderation") + // ErrBadToken means a confirmation or bounce token is invalid, used or expired. + ErrBadToken = errors.New("invalid or expired token") + // ErrJoinClosed means the list does not accept self-service subscriptions. + ErrJoinClosed = errors.New("list is closed") ) +// Recipient is a subscriber that receives a message. +type Recipient struct { + ID int64 // member id, used for the VERP bounce address + Address string +} + // Result describes how to deliver an accepted message. type Result struct { List store.List - Recipients []string // subscriber addresses - Data []byte // message to send + Recipients []Recipient + Data []byte // message to send, with list headers } // Router applies list policy to incoming messages. @@ -26,12 +43,17 @@ type Router struct { store *store.Store // BaseURL is the public web URL; enables one-click unsubscribe when set. BaseURL string + + secretOnce sync.Once + secretVal []byte + secretErr error } // New creates a Router backed by s. func New(s *store.Store) *Router { return &Router{store: s} } // Route validates a message from sender to the list address rcpt. +// Moderated lists queue the message and return ErrHeld. func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) { l, err := r.store.FindList(rcpt) if errors.Is(err, store.ErrNotFound) { @@ -40,6 +62,12 @@ func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) { if err != nil { return Result{}, err } + if l.PostPolicy == store.PostModerated { + if _, err := r.store.HoldMessage(l.ID, sender, data); err != nil { + return Result{}, err + } + return Result{}, ErrHeld + } ok, err := r.store.IsMember(l.ID, sender) if err != nil { return Result{}, err @@ -47,13 +75,18 @@ func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) { if !ok { return Result{}, ErrNotMember } - members, err := r.store.Members(l.ID) + return r.prepare(l, data) +} + +// prepare builds the delivery plan for a list. +func (r *Router) prepare(l store.List, data []byte) (Result, error) { + members, err := r.store.ActiveMembers(l.ID) if err != nil { return Result{}, err } - rcpts := make([]string, len(members)) + rcpts := make([]Recipient, len(members)) for i, m := range members { - rcpts[i] = m.Address + rcpts[i] = Recipient{ID: m.ID, Address: m.Address} } out, err := rewrite(l, r.BaseURL, data) if err != nil { @@ -61,3 +94,44 @@ func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) { } return Result{List: l, Recipients: rcpts, Data: out}, nil } + +// Send delivers the message to every recipient, each with its own signed +// bounce address (VERP). Individual failures are logged and skipped; an error is +// returned only when nobody could be reached, so a retry cannot duplicate mail. +func (r *Router) Send(ctx context.Context, out outbound.Sender, res Result) error { + failed := 0 + for _, rc := range res.Recipients { + from := r.bounceAddress(res.List, rc.ID) + if from == "" { + return errors.New("cannot create bounce address") + } + if err := out.Send(ctx, from, []string{rc.Address}, res.Data); err != nil { + failed++ + log.Printf("deliver %s to %s: %v", res.List.Address(), rc.Address, err) + } + } + if failed > 0 && failed == len(res.Recipients) { + return fmt.Errorf("delivery failed for all %d recipients", failed) + } + return nil +} + +// Release approves a held message and delivers it. +func (r *Router) Release(ctx context.Context, out outbound.Sender, heldID int64) error { + h, err := r.store.GetHeld(heldID) + if err != nil { + return err + } + l, err := r.store.GetList(h.ListID) + if err != nil { + return err + } + res, err := r.prepare(l, h.Data) + if err != nil { + return err + } + if err := r.Send(ctx, out, res); err != nil { + return err + } + return r.store.DeleteHeld(heldID) +} diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go new file mode 100644 index 0000000..67cd590 --- /dev/null +++ b/internal/store/m2_test.go @@ -0,0 +1,185 @@ +package store + +import ( + "database/sql" + "errors" + "path/filepath" + "testing" + "time" +) + +func TestListPolicies(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if l.PostPolicy != "members" || l.JoinPolicy != "open" { + t.Fatalf("defaults: %+v", l) + } + if err := s.SetPolicies(l.ID, "moderated", "closed"); err != nil { + t.Fatal(err) + } + got, _ := s.GetList(l.ID) + if got.PostPolicy != "moderated" || got.JoinPolicy != "closed" { + t.Fatalf("policies: %+v", got) + } + if err := s.SetPolicies(l.ID, "bogus", "open"); err == nil { + t.Fatal("invalid policy must be rejected") + } + if err := s.SetPolicies(999, "members", "open"); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestHeldQueue(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + id, err := s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: hi\r\n\r\nx")) + if err != nil { + t.Fatal(err) + } + hs, err := s.ListHeld(l.ID) + if err != nil || len(hs) != 1 || hs[0].ID != id || hs[0].Sender != "bob@x.com" { + t.Fatalf("held: %+v %v", hs, err) + } + h, err := s.GetHeld(id) + if err != nil || string(h.Data) != "Subject: hi\r\n\r\nx" || h.ListID != l.ID { + t.Fatalf("get: %+v %v", h, err) + } + if err := s.DeleteHeld(id); err != nil { + t.Fatal(err) + } + if _, err := s.GetHeld(id); !errors.Is(err, ErrNotFound) { + t.Fatal("held message should be gone") + } + if err := s.DeleteHeld(id); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestPendingTokens(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + + tok, created, err := s.CreatePending(l.ID, "A@x.com", "join") + if err != nil || !created || tok == "" { + t.Fatalf("create: %q %v %v", tok, created, err) + } + // Rate limit: same request within an hour is not recreated. + now = now.Add(30 * time.Minute) + if _, created, _ := s.CreatePending(l.ID, "a@x.com", "join"); created { + t.Fatal("must be rate limited") + } + // A different action is independent. + if _, created, _ := s.CreatePending(l.ID, "a@x.com", "leave"); !created { + t.Fatal("leave is a separate request") + } + p, err := s.ConfirmPending(tok) + if err != nil || p.Address != "a@x.com" || p.Action != "join" || p.ListID != l.ID { + t.Fatalf("confirm: %+v %v", p, err) + } + if _, err := s.ConfirmPending(tok); !errors.Is(err, ErrNotFound) { + t.Fatal("token is single use") + } + if _, err := s.ConfirmPending("nope"); !errors.Is(err, ErrNotFound) { + t.Fatal("unknown token") + } + // Expiry. + tok2, _, _ := s.CreatePending(l.ID, "b@x.com", "join") + now = now.Add(73 * time.Hour) + if _, err := s.ConfirmPending(tok2); !errors.Is(err, ErrNotFound) { + t.Fatal("token must expire after 72h") + } +} + +func TestSecretIsStable(t *testing.T) { + s := newTestStore(t) + a, err := s.Secret() + if err != nil || len(a) < 32 { + t.Fatalf("secret: %x %v", a, err) + } + b, _ := s.Secret() + if string(a) != string(b) { + t.Fatal("secret must not change") + } +} + +func TestBounceDisablesAfterThreshold(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "a@x.com", "A") + ms, _ := s.Members(l.ID) + id := ms[0].ID + + for i := 1; i <= 2; i++ { + disabled, err := s.RecordHardBounce(id, 3) + if err != nil || disabled { + t.Fatalf("bounce %d: %v %v", i, disabled, err) + } + } + disabled, _ := s.RecordHardBounce(id, 3) + if !disabled { + t.Fatal("third hard bounce must disable") + } + ms, _ = s.Members(l.ID) + if !ms[0].Disabled || ms[0].BounceCount != 3 { + t.Fatalf("member: %+v", ms[0]) + } + if act, _ := s.ActiveMembers(l.ID); len(act) != 0 { + t.Fatalf("active: %+v", act) + } + // Re-adding resets the state. + s.AddMember(l.ID, "a@x.com", "A") + act, _ := s.ActiveMembers(l.ID) + if len(act) != 1 || act[0].BounceCount != 0 || act[0].ID != id { + t.Fatalf("after re-add: %+v", act) + } + m, err := s.MemberByID(id) + if err != nil || m.Address != "a@x.com" || m.ListID != l.ID { + t.Fatalf("by id: %+v %v", m, err) + } + if _, err := s.MemberByID(999); !errors.Is(err, ErrNotFound) { + t.Fatal("unknown id") + } + if _, err := s.RecordHardBounce(999, 3); !errors.Is(err, ErrNotFound) { + t.Fatal("unknown id") + } +} + +func TestMigrateFromM1(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(` +CREATE TABLE lists (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, domain TEXT NOT NULL, title TEXT NOT NULL DEFAULT '', UNIQUE(name, domain)); +CREATE TABLE members (list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, address TEXT NOT NULL, display_name TEXT NOT NULL DEFAULT '', PRIMARY KEY(list_id, address)); +INSERT INTO lists(name, domain, title) VALUES('dev','example.org','Dev'); +INSERT INTO members(list_id, address, display_name) VALUES(1,'a@x.com','A'),(1,'b@x.com','B');`) + if err != nil { + t.Fatal(err) + } + db.Close() + + s, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s.Close() + l, err := s.FindList("dev@example.org") + if err != nil || l.PostPolicy != "members" || l.JoinPolicy != "open" { + t.Fatalf("list: %+v %v", l, err) + } + ms, _ := s.Members(l.ID) + if len(ms) != 2 || ms[0].ID == 0 || ms[0].Address != "a@x.com" { + t.Fatalf("members: %+v", ms) + } + s.Close() + // Opening again is a no-op. + s2, err := Open(path) + if err != nil { + t.Fatal(err) + } + s2.Close() +} diff --git a/internal/store/store.go b/internal/store/store.go index 9bf527f..e74e487 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1,27 +1,52 @@ -// Package store persists lists and members in SQLite. +// Package store persists lists, members and moderation state in SQLite. package store import ( + "crypto/rand" "database/sql" + "encoding/hex" "errors" "fmt" "strings" + "time" _ "modernc.org/sqlite" ) -// ErrNotFound is returned when a list or member does not exist. -var ErrNotFound = errors.New("not found") +var ( + // ErrNotFound is returned when a record does not exist. + ErrNotFound = errors.New("not found") + // ErrExists is returned when creating something that already exists. + ErrExists = errors.New("already exists") +) -// ErrExists is returned when creating something that already exists. -var ErrExists = errors.New("already exists") +// Policy values. +const ( + PostMembers = "members" // only members may post, others are rejected + PostModerated = "moderated" // every post waits for approval + JoinOpen = "open" // anyone can subscribe (with confirmation) + JoinClosed = "closed" // only the admin adds members +) + +// Pending actions. +const ( + ActionJoin = "join" + ActionLeave = "leave" +) + +const ( + pendingTTL = 72 * time.Hour + pendingRateLimit = time.Hour +) // List is a mailing list. type List struct { - ID int64 - Name string // local part, lowercase - Domain string // lowercase - Title string + ID int64 + Name string // local part, lowercase + Domain string // lowercase + Title string + PostPolicy string + JoinPolicy string } // Address returns the list posting address. @@ -29,26 +54,74 @@ func (l List) Address() string { return l.Name + "@" + l.Domain } // Member is a list subscriber. type Member struct { + ID int64 + ListID int64 Address string // lowercase DisplayName string + Disabled bool // too many hard bounces: no more delivery + BounceCount int +} + +// Held is a message waiting for moderation. +type Held struct { + ID int64 + ListID int64 + Sender string + Data []byte + CreatedAt time.Time +} + +// Pending is an unconfirmed subscribe or unsubscribe request. +type Pending struct { + Token string + ListID int64 + Address string + Action string } // Store wraps the SQLite database. -type Store struct{ db *sql.DB } +type Store struct { + db *sql.DB + // Now returns the current time; tests replace it. + Now func() time.Time +} const schema = ` CREATE TABLE IF NOT EXISTS lists ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - domain TEXT NOT NULL, - title TEXT NOT NULL DEFAULT '', + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + domain TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + post_policy TEXT NOT NULL DEFAULT 'members', + join_policy TEXT NOT NULL DEFAULT 'open', UNIQUE(name, domain) ); CREATE TABLE IF NOT EXISTS members ( + id INTEGER PRIMARY KEY AUTOINCREMENT, list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, address TEXT NOT NULL, display_name TEXT NOT NULL DEFAULT '', - PRIMARY KEY(list_id, address) + disabled INTEGER NOT NULL DEFAULT 0, + bounce_count INTEGER NOT NULL DEFAULT 0, + UNIQUE(list_id, address) +); +CREATE TABLE IF NOT EXISTS held ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + sender TEXT NOT NULL, + data BLOB NOT NULL, + created_at INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS pending ( + token TEXT PRIMARY KEY, + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + address TEXT NOT NULL, + action TEXT NOT NULL, + created_at INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL );` // Open opens (and migrates) the database at path. Use ":memory:" for tests. @@ -63,106 +136,60 @@ func Open(path string) (*Store, error) { db.Close() return nil, err } - if _, err := db.Exec(schema); err != nil { + if err := migrate(db); err != nil { db.Close() return nil, fmt.Errorf("migrate: %w", err) } - return &Store{db: db}, nil + return &Store{db: db, Now: time.Now}, nil +} + +func hasColumn(db *sql.DB, table, column string) (bool, error) { + var n int + err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info(?) WHERE name=?`, table, column).Scan(&n) + return n > 0, err +} + +// migrate upgrades a database created by milestone 1, then ensures the schema. +func migrate(db *sql.DB) error { + if ok, err := hasColumn(db, "members", "list_id"); err != nil { + return err + } else if ok { + if hasID, err := hasColumn(db, "members", "id"); err != nil { + return err + } else if !hasID { + if _, err := db.Exec(`ALTER TABLE members RENAME TO members_v1`); err != nil { + return err + } + if _, err := db.Exec(schema); err != nil { + return err + } + if _, err := db.Exec(`INSERT INTO members(list_id, address, display_name) + SELECT list_id, address, display_name FROM members_v1 ORDER BY list_id, address; + DROP TABLE members_v1;`); err != nil { + return err + } + } + } + if ok, err := hasColumn(db, "lists", "name"); err != nil { + return err + } else if ok { + if has, _ := hasColumn(db, "lists", "post_policy"); !has { + if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN post_policy TEXT NOT NULL DEFAULT 'members'; + ALTER TABLE lists ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'open';`); err != nil { + return err + } + } + } + _, err := db.Exec(schema) + return err } // Close closes the database. func (s *Store) Close() error { return s.db.Close() } -// CreateList adds a new list. -func (s *Store) CreateList(name, domain, title string) (List, error) { - l := List{Name: strings.ToLower(name), Domain: strings.ToLower(domain), Title: title} - res, err := s.db.Exec(`INSERT INTO lists(name, domain, title) VALUES(?,?,?)`, l.Name, l.Domain, l.Title) - if err != nil { - if strings.Contains(err.Error(), "UNIQUE") { - return List{}, ErrExists - } - return List{}, err - } - l.ID, _ = res.LastInsertId() - return l, nil -} +func norm(addr string) string { return strings.ToLower(strings.TrimSpace(addr)) } -// FindList resolves a posting address like "dev@example.org". -func (s *Store) FindList(addr string) (List, error) { - name, domain, ok := strings.Cut(strings.ToLower(strings.TrimSpace(addr)), "@") - if !ok { - return List{}, ErrNotFound - } - var l List - err := s.db.QueryRow(`SELECT id, name, domain, title FROM lists WHERE name=? AND domain=?`, name, domain). - Scan(&l.ID, &l.Name, &l.Domain, &l.Title) - if errors.Is(err, sql.ErrNoRows) { - return List{}, ErrNotFound - } - return l, err -} - -// AddMember subscribes an address to a list (idempotent). -func (s *Store) AddMember(listID int64, addr, displayName string) error { - _, err := s.db.Exec(`INSERT OR REPLACE INTO members(list_id, address, display_name) VALUES(?,?,?)`, - listID, strings.ToLower(strings.TrimSpace(addr)), displayName) - return err -} - -// IsMember reports whether addr is subscribed to the list. -func (s *Store) IsMember(listID int64, addr string) (bool, error) { - var n int - err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=?`, - listID, strings.ToLower(strings.TrimSpace(addr))).Scan(&n) - return n > 0, err -} - -// Members returns all subscribers of a list. -func (s *Store) Members(listID int64) ([]Member, error) { - rows, err := s.db.Query(`SELECT address, display_name FROM members WHERE list_id=? ORDER BY address`, listID) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Member - for rows.Next() { - var m Member - if err := rows.Scan(&m.Address, &m.DisplayName); err != nil { - return nil, err - } - out = append(out, m) - } - return out, rows.Err() -} - -// ListLists returns every list ordered by address. -func (s *Store) ListLists() ([]List, error) { - rows, err := s.db.Query(`SELECT id, name, domain, title FROM lists ORDER BY domain, name`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []List - for rows.Next() { - var l List - if err := rows.Scan(&l.ID, &l.Name, &l.Domain, &l.Title); err != nil { - return nil, err - } - out = append(out, l) - } - return out, rows.Err() -} - -// GetList returns a list by id. -func (s *Store) GetList(id int64) (List, error) { - var l List - err := s.db.QueryRow(`SELECT id, name, domain, title FROM lists WHERE id=?`, id). - Scan(&l.ID, &l.Name, &l.Domain, &l.Title) - if errors.Is(err, sql.ErrNoRows) { - return List{}, ErrNotFound - } - return l, err -} +func isUnique(err error) bool { return err != nil && strings.Contains(err.Error(), "UNIQUE") } func affected(res sql.Result, err error) error { if err != nil { @@ -174,18 +201,287 @@ func affected(res sql.Result, err error) error { return nil } +// Lists. + +const listCols = `id, name, domain, title, post_policy, join_policy` + +func scanList(row interface{ Scan(...any) error }) (List, error) { + var l List + err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy) + if errors.Is(err, sql.ErrNoRows) { + return List{}, ErrNotFound + } + return l, err +} + +// CreateList adds a new list with default policies. +func (s *Store) CreateList(name, domain, title string) (List, error) { + res, err := s.db.Exec(`INSERT INTO lists(name, domain, title) VALUES(?,?,?)`, + strings.ToLower(name), strings.ToLower(domain), title) + if isUnique(err) { + return List{}, ErrExists + } + if err != nil { + return List{}, err + } + id, _ := res.LastInsertId() + return s.GetList(id) +} + +// FindList resolves a posting address like "dev@example.org". +func (s *Store) FindList(addr string) (List, error) { + name, domain, ok := strings.Cut(norm(addr), "@") + if !ok { + return List{}, ErrNotFound + } + return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE name=? AND domain=?`, name, domain)) +} + +// GetList returns a list by id. +func (s *Store) GetList(id int64) (List, error) { + return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE id=?`, id)) +} + +// ListLists returns every list ordered by address. +func (s *Store) ListLists() ([]List, error) { + rows, err := s.db.Query(`SELECT ` + listCols + ` FROM lists ORDER BY domain, name`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []List + for rows.Next() { + l, err := scanList(rows) + if err != nil { + return nil, err + } + out = append(out, l) + } + return out, rows.Err() +} + // UpdateList changes the title of a list. func (s *Store) UpdateList(id int64, title string) error { return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id)) } -// DeleteList removes a list and its members. +// SetPolicies changes the post and join policies of a list. +func (s *Store) SetPolicies(id int64, post, join string) error { + if post != PostMembers && post != PostModerated { + return fmt.Errorf("invalid post policy %q", post) + } + if join != JoinOpen && join != JoinClosed { + return fmt.Errorf("invalid join policy %q", join) + } + return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=? WHERE id=?`, post, join, id)) +} + +// DeleteList removes a list with its members, held messages and pending requests. func (s *Store) DeleteList(id int64) error { return affected(s.db.Exec(`DELETE FROM lists WHERE id=?`, id)) } +// Members. + +const memberCols = `id, list_id, address, display_name, disabled, bounce_count` + +func scanMember(row interface{ Scan(...any) error }) (Member, error) { + var m Member + var dis int + err := row.Scan(&m.ID, &m.ListID, &m.Address, &m.DisplayName, &dis, &m.BounceCount) + if errors.Is(err, sql.ErrNoRows) { + return Member{}, ErrNotFound + } + m.Disabled = dis != 0 + return m, err +} + +// AddMember subscribes an address to a list. Re-adding keeps the member id and +// clears the bounce state. +func (s *Store) AddMember(listID int64, addr, displayName string) error { + _, err := s.db.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?) + ON CONFLICT(list_id, address) DO UPDATE SET display_name=excluded.display_name, disabled=0, bounce_count=0`, + listID, norm(addr), displayName) + return err +} + +// IsMember reports whether addr is subscribed to the list. +func (s *Store) IsMember(listID int64, addr string) (bool, error) { + var n int + err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=?`, listID, norm(addr)).Scan(&n) + return n > 0, err +} + +func (s *Store) queryMembers(where string, args ...any) ([]Member, error) { + rows, err := s.db.Query(`SELECT `+memberCols+` FROM members WHERE `+where+` ORDER BY address`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Member + for rows.Next() { + m, err := scanMember(rows) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} + +// Members returns all subscribers of a list, including disabled ones. +func (s *Store) Members(listID int64) ([]Member, error) { + return s.queryMembers(`list_id=?`, listID) +} + +// ActiveMembers returns the subscribers that should receive mail. +func (s *Store) ActiveMembers(listID int64) ([]Member, error) { + return s.queryMembers(`list_id=? AND disabled=0`, listID) +} + +// MemberByID returns a member by its id. +func (s *Store) MemberByID(id int64) (Member, error) { + return scanMember(s.db.QueryRow(`SELECT `+memberCols+` FROM members WHERE id=?`, id)) +} + // RemoveMember unsubscribes an address from a list. func (s *Store) RemoveMember(listID int64, addr string) error { - return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, - listID, strings.ToLower(strings.TrimSpace(addr)))) + return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr))) +} + +// RecordHardBounce counts a hard bounce and disables the member once the count +// reaches threshold. It reports whether the member is now disabled. +func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) { + if err := affected(s.db.Exec(`UPDATE members SET bounce_count=bounce_count+1 WHERE id=?`, memberID)); err != nil { + return false, err + } + if _, err := s.db.Exec(`UPDATE members SET disabled=1 WHERE id=? AND bounce_count>=?`, memberID, threshold); err != nil { + return false, err + } + m, err := s.MemberByID(memberID) + return m.Disabled, err +} + +// Held messages. + +// HoldMessage queues a message for moderation and returns its id. +func (s *Store) HoldMessage(listID int64, sender string, data []byte) (int64, error) { + res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at) VALUES(?,?,?,?)`, + listID, norm(sender), data, s.Now().Unix()) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func scanHeld(row interface{ Scan(...any) error }) (Held, error) { + var h Held + var ts int64 + err := row.Scan(&h.ID, &h.ListID, &h.Sender, &h.Data, &ts) + if errors.Is(err, sql.ErrNoRows) { + return Held{}, ErrNotFound + } + h.CreatedAt = time.Unix(ts, 0) + return h, err +} + +// ListHeld returns the held messages of a list, oldest first. +func (s *Store) ListHeld(listID int64) ([]Held, error) { + rows, err := s.db.Query(`SELECT id, list_id, sender, data, created_at FROM held WHERE list_id=? ORDER BY id`, listID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Held + for rows.Next() { + h, err := scanHeld(rows) + if err != nil { + return nil, err + } + out = append(out, h) + } + return out, rows.Err() +} + +// GetHeld returns a held message by id. +func (s *Store) GetHeld(id int64) (Held, error) { + return scanHeld(s.db.QueryRow(`SELECT id, list_id, sender, data, created_at FROM held WHERE id=?`, id)) +} + +// DeleteHeld discards a held message. +func (s *Store) DeleteHeld(id int64) error { + return affected(s.db.Exec(`DELETE FROM held WHERE id=?`, id)) +} + +// Pending confirmations. + +func randomHex(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +// CreatePending records a subscribe or unsubscribe request and returns its +// token. created is false when an identical request was made in the last hour: +// this limits how often a third party can trigger confirmation mail. +func (s *Store) CreatePending(listID int64, addr, action string) (token string, created bool, err error) { + addr = norm(addr) + now := s.Now() + var n int + err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE list_id=? AND address=? AND action=? AND created_at>?`, + listID, addr, action, now.Add(-pendingRateLimit).Unix()).Scan(&n) + if err != nil || n > 0 { + return "", false, err + } + token, err = randomHex(16) + if err != nil { + return "", false, err + } + _, err = s.db.Exec(`INSERT INTO pending(token, list_id, address, action, created_at) VALUES(?,?,?,?,?)`, + token, listID, addr, action, now.Unix()) + if err != nil { + return "", false, err + } + return token, true, nil +} + +// ConfirmPending consumes a token. Unknown, used and expired tokens give ErrNotFound. +func (s *Store) ConfirmPending(token string) (Pending, error) { + var p Pending + var ts int64 + err := s.db.QueryRow(`SELECT token, list_id, address, action, created_at FROM pending WHERE token=?`, token). + Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts) + if errors.Is(err, sql.ErrNoRows) { + return Pending{}, ErrNotFound + } + if err != nil { + return Pending{}, err + } + if _, err := s.db.Exec(`DELETE FROM pending WHERE token=?`, token); err != nil { + return Pending{}, err + } + if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL { + return Pending{}, ErrNotFound + } + return p, nil +} + +// Secret returns the HMAC secret, generating and saving it on first use. +func (s *Store) Secret() ([]byte, error) { + var v string + err := s.db.QueryRow(`SELECT value FROM settings WHERE key='secret'`).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + v, err = randomHex(32) + if err != nil { + return nil, err + } + if _, err = s.db.Exec(`INSERT INTO settings(key, value) VALUES('secret', ?)`, v); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return hex.DecodeString(v) }