Merge pull request 'Milestone 2: moderation, subscribe/unsubscribe confirmation, VERP bounces' (#6) from feat/m2-moderation-subscribe-bounce into main

This commit is contained in:
squidd 2026-11-09 15:06:29 +01:00
commit d2adddc8a7
11 changed files with 1511 additions and 118 deletions

View file

@ -5,6 +5,10 @@ A modern, single-binary mailing list manager: an evolution of Mailman.
- Postfix delivers to parsimony over LMTP; parsimony relays through SMTP.
- Discussion lists with member-only posting, `List-*` headers, one-click unsubscribe
and DMARC-safe `From` rewriting.
- Self-service subscribe and unsubscribe by email with double opt-in:
send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation.
- Moderated lists: posts wait in a queue that the API can approve or discard.
- VERP bounces: every recipient gets a signed return address; 3 hard bounces disable a member.
- SQLite storage, one YAML config file.
- Admin REST API under `/api/v1` (see `docs/openapi.yaml`).
@ -20,3 +24,13 @@ The API is disabled when `PARSIMONY_API_TOKEN` is not set.
```
curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists
```
## Postfix
Deliver the list domains to parsimony over LMTP and keep the `+` extensions intact, for example:
```
transport_maps = hash:/etc/postfix/transport
# /etc/postfix/transport
lists.example.org lmtp:inet:127.0.0.1:2424
```

View file

@ -20,11 +20,23 @@ components:
domain: {type: string}
title: {type: string}
address: {type: string}
post_policy: {type: string, enum: [members, moderated]}
join_policy: {type: string, enum: [open, closed]}
Member:
type: object
properties:
address: {type: string}
display_name: {type: string}
disabled: {type: boolean, description: Disabled after 3 hard bounces; re-adding the member re-enables it}
bounce_count: {type: integer}
Held:
type: object
properties:
id: {type: integer}
sender: {type: string}
subject: {type: string}
size: {type: integer}
created_at: {type: string, format: date-time}
Error:
type: object
properties:
@ -85,16 +97,17 @@ paths:
schema: {$ref: '#/components/schemas/List'}
'404': {$ref: '#/components/responses/Error'}
patch:
summary: Update the list title
summary: Update title and policies (send only the fields to change)
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [title]
properties:
title: {type: string}
post_policy: {type: string, enum: [members, moderated]}
join_policy: {type: string, enum: [open, closed]}
responses:
'200':
description: OK
@ -149,3 +162,32 @@ paths:
responses:
'204': {description: Removed}
'404': {$ref: '#/components/responses/Error'}
/lists/{id}/held:
parameters:
- {name: id, in: path, required: true, schema: {type: integer}}
get:
summary: Messages waiting for moderation
responses:
'200':
description: OK
content:
application/json:
schema: {type: array, items: {$ref: '#/components/schemas/Held'}}
'404': {$ref: '#/components/responses/Error'}
/held/{id}/approve:
parameters:
- {name: id, in: path, required: true, schema: {type: integer}}
post:
summary: Approve and deliver a held message
responses:
'204': {description: Delivered}
'404': {$ref: '#/components/responses/Error'}
'501': {$ref: '#/components/responses/Error'}
/held/{id}:
parameters:
- {name: id, in: path, required: true, schema: {type: integer}}
delete:
summary: Discard a held message
responses:
'204': {description: Discarded}
'404': {$ref: '#/components/responses/Error'}

View file

@ -2,26 +2,37 @@
package api
import (
"bytes"
"context"
"crypto/subtle"
"encoding/json"
"errors"
"mime"
"net/http"
"net/mail"
"strconv"
"strings"
"time"
"codeberg.org/squidd/parsimony/internal/store"
)
// Releaser approves a held message: it delivers it to the list.
type Releaser interface {
Release(ctx context.Context, heldID int64) error
}
type server struct {
store *store.Store
token string
store *store.Store
token string
releaser Releaser
}
// New returns the API handler. Every route except /api/v1/health needs
// "Authorization: Bearer <token>". An empty token rejects all requests.
func New(s *store.Store, token string) http.Handler {
srv := &server{store: s, token: token}
// rel may be nil: approving held messages then answers 501.
func New(s *store.Store, token string, rel Releaser) http.Handler {
srv := &server{store: s, token: token, releaser: rel}
mux := http.NewServeMux()
mux.HandleFunc("GET /api/v1/health", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
@ -34,6 +45,9 @@ func New(s *store.Store, token string) http.Handler {
mux.Handle("GET /api/v1/lists/{id}/members", srv.auth(srv.listMembers))
mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember))
mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember))
mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld))
mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld))
mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld))
return mux
}
@ -57,15 +71,21 @@ type listDTO struct {
Domain string `json:"domain"`
Title string `json:"title"`
Address string `json:"address"`
PostPolicy string `json:"post_policy"`
JoinPolicy string `json:"join_policy"`
}
func toListDTO(l store.List) listDTO {
return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address()}
return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address(),
PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy}
}
type memberDTO struct {
Address string `json:"address"`
DisplayName string `json:"display_name"`
Disabled bool `json:"disabled"`
BounceCount int `json:"bounce_count"`
}
// Helpers.
@ -138,8 +158,8 @@ func (s *server) createList(w http.ResponseWriter, r *http.Request) {
return
}
in.Name, in.Domain = strings.TrimSpace(in.Name), strings.TrimSpace(in.Domain)
if in.Name == "" || in.Domain == "" || strings.ContainsAny(in.Name+in.Domain, "@ \r\n") {
writeError(w, http.StatusBadRequest, "name and domain are required and must not contain '@' or spaces")
if in.Name == "" || in.Domain == "" || strings.ContainsAny(in.Name+in.Domain, "@+ \r\n") {
writeError(w, http.StatusBadRequest, "name and domain are required and must not contain '@', '+' or spaces")
return
}
if in.Title == "" {
@ -172,18 +192,15 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) {
return
}
var in struct {
Title string `json:"title"`
Title *string `json:"title"`
PostPolicy *string `json:"post_policy"`
JoinPolicy *string `json:"join_policy"`
}
if !decodeBody(w, r, &in) {
return
}
title := strings.TrimSpace(in.Title)
if title == "" || strings.ContainsAny(title, "\r\n") {
writeError(w, http.StatusBadRequest, "title is required and must be a single line")
return
}
if err := s.store.UpdateList(id, title); err != nil {
storeError(w, err)
if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil {
writeError(w, http.StatusBadRequest, "nothing to update")
return
}
l, err := s.store.GetList(id)
@ -191,6 +208,34 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) {
storeError(w, err)
return
}
if in.Title != nil {
title := strings.TrimSpace(*in.Title)
if title == "" || strings.ContainsAny(title, "\r\n") {
writeError(w, http.StatusBadRequest, "title must be a non-empty single line")
return
}
l.Title = title
}
if in.PostPolicy != nil {
l.PostPolicy = *in.PostPolicy
}
if in.JoinPolicy != nil {
l.JoinPolicy = *in.JoinPolicy
}
// Validate before writing anything.
if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated) ||
(l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) {
writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy must be open or closed")
return
}
if err := s.store.UpdateList(id, l.Title); err != nil {
storeError(w, err)
return
}
if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy); err != nil {
storeError(w, err)
return
}
writeJSON(w, http.StatusOK, toListDTO(l))
}
@ -224,7 +269,7 @@ func (s *server) listMembers(w http.ResponseWriter, r *http.Request) {
}
out := make([]memberDTO, 0, len(ms))
for _, m := range ms {
out = append(out, memberDTO{Address: m.Address, DisplayName: m.DisplayName})
out = append(out, memberDTO{Address: m.Address, DisplayName: m.DisplayName, Disabled: m.Disabled, BounceCount: m.BounceCount})
}
writeJSON(w, http.StatusOK, out)
}
@ -269,3 +314,74 @@ func (s *server) removeMember(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusNoContent)
}
// Handlers: held messages.
type heldDTO struct {
ID int64 `json:"id"`
Sender string `json:"sender"`
Subject string `json:"subject"`
Size int `json:"size"`
CreatedAt time.Time `json:"created_at"`
}
func subjectOf(data []byte) string {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return ""
}
subj := m.Header.Get("Subject")
if dec, err := new(mime.WordDecoder).DecodeHeader(subj); err == nil {
return dec
}
return subj
}
func (s *server) listHeld(w http.ResponseWriter, r *http.Request) {
id, ok := listID(w, r)
if !ok {
return
}
if _, err := s.store.GetList(id); err != nil {
storeError(w, err)
return
}
hs, err := s.store.ListHeld(id)
if err != nil {
storeError(w, err)
return
}
out := make([]heldDTO, 0, len(hs))
for _, h := range hs {
out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: subjectOf(h.Data), Size: len(h.Data), CreatedAt: h.CreatedAt})
}
writeJSON(w, http.StatusOK, out)
}
func (s *server) approveHeld(w http.ResponseWriter, r *http.Request) {
id, ok := listID(w, r)
if !ok {
return
}
if s.releaser == nil {
writeError(w, http.StatusNotImplemented, "delivery is not configured")
return
}
if err := s.releaser.Release(r.Context(), id); err != nil {
storeError(w, err)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (s *server) discardHeld(w http.ResponseWriter, r *http.Request) {
id, ok := listID(w, r)
if !ok {
return
}
if err := s.store.DeleteHeld(id); err != nil {
storeError(w, err)
return
}
w.WriteHeader(http.StatusNoContent)
}

View file

@ -1,6 +1,7 @@
package api
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
@ -19,7 +20,7 @@ func newServer(t *testing.T) http.Handler {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
return New(s, token)
return New(s, token, nil)
}
func do(t *testing.T, h http.Handler, method, path, body string, auth bool) *httptest.ResponseRecorder {
@ -68,7 +69,7 @@ func TestAuthRequired(t *testing.T) {
func TestEmptyTokenRejectsEverything(t *testing.T) {
s, _ := store.Open(":memory:")
defer s.Close()
h := New(s, "")
h := New(s, "", nil)
req := httptest.NewRequest("GET", "/api/v1/lists", nil)
req.Header.Set("Authorization", "Bearer ")
rec := httptest.NewRecorder()
@ -77,11 +78,13 @@ func TestEmptyTokenRejectsEverything(t *testing.T) {
}
type listJSON struct {
ID int64 `json:"id"`
Name string `json:"name"`
Domain string `json:"domain"`
Title string `json:"title"`
Address string `json:"address"`
ID int64 `json:"id"`
Name string `json:"name"`
Domain string `json:"domain"`
Title string `json:"title"`
Address string `json:"address"`
PostPolicy string `json:"post_policy"`
JoinPolicy string `json:"join_policy"`
}
func createList(t *testing.T, h http.Handler) listJSON {
@ -141,6 +144,7 @@ func TestCreateListValidation(t *testing.T) {
`{}`,
`{"name":"","domain":"example.org"}`,
`{"name":"a@b","domain":"example.org"}`,
`{"name":"a+b","domain":"example.org"}`,
`{"name":"dev","domain":""}`,
`{"name":"dev","domain":"example.org","bogus":1}`,
} {
@ -159,6 +163,8 @@ func TestCreateListValidation(t *testing.T) {
type memberJSON struct {
Address string `json:"address"`
DisplayName string `json:"display_name"`
Disabled bool `json:"disabled"`
BounceCount int `json:"bounce_count"`
}
func TestMembers(t *testing.T) {
@ -194,3 +200,118 @@ func TestMembers(t *testing.T) {
expect(t, do(t, h, "DELETE", "/api/v1/lists/1/members/alice%40example.com", "", true), 204)
expect(t, do(t, h, "DELETE", "/api/v1/lists/1/members/alice%40example.com", "", true), 404)
}
func TestListPoliciesViaAPI(t *testing.T) {
h := newServer(t)
l := createList(t, h)
if l.PostPolicy != "members" || l.JoinPolicy != "open" {
t.Fatalf("defaults: %+v", l)
}
rec := do(t, h, "PATCH", "/api/v1/lists/1", `{"post_policy":"moderated"}`, true)
expect(t, rec, 200)
var u listJSON
decode(t, rec, &u)
if u.PostPolicy != "moderated" || u.JoinPolicy != "open" || u.Title != "Dev" {
t.Fatalf("patched: %+v", u)
}
expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"join_policy":"closed","title":"New"}`, true), 200)
rec = do(t, h, "GET", "/api/v1/lists/1", "", true)
decode(t, rec, &u)
if u.JoinPolicy != "closed" || u.Title != "New" || u.PostPolicy != "moderated" {
t.Fatalf("after: %+v", u)
}
expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"post_policy":"bogus"}`, true), 400)
}
func TestMemberBounceState(t *testing.T) {
s, _ := store.Open(":memory:")
defer s.Close()
h := New(s, token, nil)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.AddMember(l.ID, "a@x.com", "A")
ms, _ := s.Members(l.ID)
for i := 0; i < 3; i++ {
s.RecordHardBounce(ms[0].ID, 3)
}
rec := do(t, h, "GET", "/api/v1/lists/1/members", "", true)
var out []memberJSON
decode(t, rec, &out)
if len(out) != 1 || !out[0].Disabled || out[0].BounceCount != 3 {
t.Fatalf("members: %+v", out)
}
// Re-adding re-enables.
expect(t, do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com"}`, true), 201)
rec = do(t, h, "GET", "/api/v1/lists/1/members", "", true)
decode(t, rec, &out)
if out[0].Disabled || out[0].BounceCount != 0 {
t.Fatalf("after re-add: %+v", out)
}
}
type fakeReleaser struct{ released []int64 }
func (f *fakeReleaser) Release(_ context.Context, id int64) error {
f.released = append(f.released, id)
return nil
}
func heldServer(t *testing.T) (http.Handler, *store.Store, *fakeReleaser) {
t.Helper()
s, _ := store.Open(":memory:")
t.Cleanup(func() { s.Close() })
l, _ := s.CreateList("dev", "example.org", "Dev")
s.HoldMessage(l.ID, "Bob@x.com", []byte("From: Bob <bob@x.com>\r\nSubject: Hello there\r\n\r\nbody"))
rel := &fakeReleaser{}
return New(s, token, rel), s, rel
}
type heldJSON struct {
ID int64 `json:"id"`
Sender string `json:"sender"`
Subject string `json:"subject"`
Size int `json:"size"`
}
func TestHeldQueueAPI(t *testing.T) {
h, _, rel := heldServer(t)
rec := do(t, h, "GET", "/api/v1/lists/1/held", "", true)
expect(t, rec, 200)
var hs []heldJSON
decode(t, rec, &hs)
if len(hs) != 1 || hs[0].Sender != "bob@x.com" || hs[0].Subject != "Hello there" || hs[0].Size == 0 {
t.Fatalf("held: %+v", hs)
}
expect(t, do(t, h, "GET", "/api/v1/lists/99/held", "", true), 404)
expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", true), 204)
if len(rel.released) != 1 || rel.released[0] != 1 {
t.Fatalf("released: %v", rel.released)
}
expect(t, do(t, h, "DELETE", "/api/v1/held/1", "", true), 204)
expect(t, do(t, h, "DELETE", "/api/v1/held/1", "", true), 404)
expect(t, do(t, h, "DELETE", "/api/v1/held/abc", "", true), 404)
}
func TestHeldRequiresAuth(t *testing.T) {
h, _, _ := heldServer(t)
expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", false), 401)
expect(t, do(t, h, "GET", "/api/v1/lists/1/held", "", false), 401)
}
func TestApproveWithoutReleaser(t *testing.T) {
s, _ := store.Open(":memory:")
defer s.Close()
h := New(s, token, nil)
expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", true), 501)
}
func TestApproveUnknownHeld(t *testing.T) {
s, _ := store.Open(":memory:")
defer s.Close()
h := New(s, token, errReleaser{store.ErrNotFound})
expect(t, do(t, h, "POST", "/api/v1/held/5/approve", "", true), 404)
}
type errReleaser struct{ err error }
func (e errReleaser) Release(context.Context, int64) error { return e.err }

View file

@ -77,22 +77,33 @@ func (s *session) LMTPData(r io.Reader, status smtp.StatusCollector) error {
}
func (s *session) deliver(ctx context.Context, rcpt string, data []byte) error {
res, err := s.b.router.Route(s.from, rcpt, data)
rt := s.b.router
if handled, err := rt.Command(ctx, s.b.out, s.from, rcpt, data); handled {
return s.status(rcpt, err)
}
res, err := rt.Route(s.from, rcpt, data)
if err != nil {
return s.status(rcpt, err)
}
return s.status(rcpt, rt.Send(ctx, s.b.out, res))
}
// status turns a router result into an LMTP reply.
func (s *session) status(rcpt string, err error) error {
switch {
case err == nil, errors.Is(err, router.ErrHeld):
return nil
case errors.Is(err, router.ErrUnknownList):
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such list"}
case errors.Is(err, router.ErrNotMember):
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "only list members may post"}
case err != nil:
log.Printf("route %s: %v", rcpt, err)
return tempFail()
case errors.Is(err, router.ErrBadToken):
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "invalid or expired token"}
case errors.Is(err, router.ErrJoinClosed):
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "this list does not accept subscriptions"}
}
bounce := res.List.Name + "+bounce@" + res.List.Domain
if err := s.b.out.Send(ctx, bounce, res.Recipients, res.Data); err != nil {
log.Printf("relay %s: %v", rcpt, err)
return tempFail()
}
return nil
log.Printf("deliver %s: %v", rcpt, err)
return tempFail()
}
func tempFail() error {

View file

@ -85,14 +85,14 @@ func TestDeliverFansOut(t *testing.T) {
if res["dev@example.org"] != nil {
t.Fatalf("unexpected error: %v", res)
}
if len(relay.Sent) != 1 {
t.Fatalf("want 1 send, got %d", len(relay.Sent))
if len(relay.Sent) != 2 {
t.Fatalf("want one send per member, got %d", len(relay.Sent))
}
s := relay.Sent[0]
if len(s.To) != 2 {
if len(s.To) != 1 {
t.Errorf("recipients = %v", s.To)
}
if s.From != "dev+bounce@example.org" {
if !strings.HasPrefix(s.From, "dev+bounce-") {
t.Errorf("envelope from = %q", s.From)
}
if !strings.Contains(string(s.Data), "List-Id: Dev <dev.example.org>") {
@ -127,7 +127,7 @@ func TestMixedRecipients(t *testing.T) {
if res["dev@example.org"] != nil || res["nope@example.org"] == nil {
t.Fatalf("results: %v", res)
}
if len(relay.Sent) != 1 {
if len(relay.Sent) != 2 {
t.Fatalf("sent = %d", len(relay.Sent))
}
}
@ -154,3 +154,47 @@ func TestRelayFailureIsTemporary(t *testing.T) {
t.Fatalf("want 451, got %v", res["dev@example.org"])
}
}
func TestJoinCommandOverLMTP(t *testing.T) {
addr, relay := setup(t)
res := deliver(t, addr, "carol@x.com", []string{"dev+join@example.org"})
if res["dev+join@example.org"] != nil {
t.Fatalf("join: %v", res)
}
if len(relay.Sent) != 1 || relay.Sent[0].To[0] != "carol@x.com" {
t.Fatalf("confirmation mail: %+v", relay.Sent)
}
}
func TestBadTokenRejected(t *testing.T) {
addr, _ := setup(t)
res := deliver(t, addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"})
var se *smtp.SMTPError
if !errors.As(res["dev+confirm-deadbeef@example.org"], &se) || se.Code != 550 {
t.Fatalf("want 550, got %v", res)
}
}
func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) {
s, _ := store.Open(":memory:")
t.Cleanup(func() { s.Close() })
l, _ := s.CreateList("dev", "example.org", "Dev")
s.AddMember(l.ID, "alice@example.com", "Alice")
s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen)
relay := &outbound.Fake{}
srv := NewServer(router.New(s), relay)
ln, _ := net.Listen("tcp", "127.0.0.1:0")
go srv.Serve(ln)
t.Cleanup(func() { srv.Close() })
res := deliver(t, ln.Addr().String(), "stranger@x.com", []string{"dev@example.org"})
if res["dev@example.org"] != nil {
t.Fatalf("held mail must be accepted: %v", res)
}
if len(relay.Sent) != 0 {
t.Fatal("nothing may be relayed")
}
if h, _ := s.ListHeld(l.ID); len(h) != 1 {
t.Fatalf("held = %d", len(h))
}
}

174
internal/router/commands.go Normal file
View file

@ -0,0 +1,174 @@
package router
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"mime"
"net/mail"
"regexp"
"strconv"
"strings"
"time"
"codeberg.org/squidd/parsimony/internal/outbound"
"codeberg.org/squidd/parsimony/internal/store"
)
// bounceThreshold is the number of hard bounces after which a member is disabled.
const bounceThreshold = 3
var statusRe = regexp.MustCompile(`(?mi)^Status:\s*([245])\.\d+\.\d+`)
func (r *Router) secret() ([]byte, error) {
r.secretOnce.Do(func() { r.secretVal, r.secretErr = r.store.Secret() })
return r.secretVal, r.secretErr
}
func (r *Router) mac(memberID int64) string {
key, err := r.secret()
if err != nil {
return ""
}
h := hmac.New(sha256.New, key)
fmt.Fprintf(h, "bounce:%d", memberID)
return hex.EncodeToString(h.Sum(nil))[:12]
}
// bounceAddress returns the VERP envelope sender for a member, or "" on error.
func (r *Router) bounceAddress(l store.List, memberID int64) string {
mac := r.mac(memberID)
if mac == "" {
return ""
}
return fmt.Sprintf("%s+bounce-%d-%s@%s", l.Name, memberID, mac, l.Domain)
}
// Command handles list command addresses such as dev+join@example.org.
// handled is false when rcpt is a plain list address (use Route for those).
func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt string, data []byte) (handled bool, err error) {
local, domain, ok := strings.Cut(strings.ToLower(strings.TrimSpace(rcpt)), "@")
if !ok {
return false, nil
}
name, tag, hasTag := strings.Cut(local, "+")
if !hasTag {
return false, nil
}
l, err := r.store.FindList(name + "@" + domain)
if errors.Is(err, store.ErrNotFound) {
return true, ErrUnknownList
}
if err != nil {
return true, err
}
switch {
case tag == "join":
return true, r.join(ctx, out, l, sender)
case tag == "leave":
return true, r.leave(ctx, out, l, sender)
case strings.HasPrefix(tag, "confirm-"):
return true, r.confirm(l, strings.TrimPrefix(tag, "confirm-"))
case tag == "bounce":
return true, nil // system mail without a token: nothing to learn from it
case strings.HasPrefix(tag, "bounce-"):
return true, r.bounce(l, strings.TrimPrefix(tag, "bounce-"), data)
}
return true, ErrUnknownList
}
func (r *Router) join(ctx context.Context, out outbound.Sender, l store.List, sender string) error {
if sender == "" {
return nil
}
if l.JoinPolicy != store.JoinOpen {
return ErrJoinClosed
}
if ok, err := r.store.IsMember(l.ID, sender); err != nil || ok {
return err // members get no mail: it would leak membership
}
return r.askConfirmation(ctx, out, l, sender, store.ActionJoin)
}
func (r *Router) leave(ctx context.Context, out outbound.Sender, l store.List, sender string) error {
if sender == "" {
return nil
}
if ok, err := r.store.IsMember(l.ID, sender); err != nil || !ok {
return err
}
return r.askConfirmation(ctx, out, l, sender, store.ActionLeave)
}
func (r *Router) askConfirmation(ctx context.Context, out outbound.Sender, l store.List, addr, action string) error {
token, created, err := r.store.CreatePending(l.ID, addr, action)
if err != nil || !created {
return err
}
verb, subject := "join", "Confirm your subscription to "+l.Title
if action == store.ActionLeave {
verb, subject = "leave", "Confirm your unsubscription from "+l.Title
}
msg := fmt.Sprintf("From: %s\r\nTo: %s\r\nReply-To: %s+confirm-%s@%s\r\nSubject: %s\r\nDate: %s\r\n"+
"Message-ID: <%s.%s@%s>\r\nAuto-Submitted: auto-generated\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+
"Someone, probably you, asked to %s %s.\r\n\r\n"+
"To confirm, just reply to this message. The request expires in 72 hours.\r\n"+
"If you did not ask for this, ignore this message and nothing will happen.\r\n",
(&mail.Address{Name: l.Title, Address: l.Address()}).String(),
(&mail.Address{Address: addr}).String(),
l.Name, token, l.Domain,
mime.QEncoding.Encode("utf-8", subject),
time.Now().Format(time.RFC1123Z),
token[:8], strconv.FormatInt(time.Now().UnixNano(), 36), l.Domain,
verb, l.Address())
return out.Send(ctx, l.Name+"+bounce@"+l.Domain, []string{addr}, []byte(msg))
}
func (r *Router) confirm(l store.List, token string) error {
p, err := r.store.ConfirmPending(token)
if errors.Is(err, store.ErrNotFound) || (err == nil && p.ListID != l.ID) {
return ErrBadToken
}
if err != nil {
return err
}
switch p.Action {
case store.ActionJoin:
if l.JoinPolicy != store.JoinOpen {
return ErrJoinClosed
}
return r.store.AddMember(l.ID, p.Address, "")
case store.ActionLeave:
if err := r.store.RemoveMember(l.ID, p.Address); err != nil && !errors.Is(err, store.ErrNotFound) {
return err
}
}
return nil
}
func (r *Router) bounce(l store.List, tag string, data []byte) error {
idStr, mac, ok := strings.Cut(tag, "-")
id, err := strconv.ParseInt(idStr, 10, 64)
if !ok || err != nil {
return ErrBadToken
}
want := r.mac(id)
if want == "" || !hmac.Equal([]byte(mac), []byte(want)) {
return ErrBadToken
}
m, err := r.store.MemberByID(id)
if errors.Is(err, store.ErrNotFound) || (err == nil && m.ListID != l.ID) {
return ErrBadToken
}
if err != nil {
return err
}
if sm := statusRe.FindSubmatch(data); sm != nil && string(sm[1]) == "5" {
_, err = r.store.RecordHardBounce(m.ID, bounceThreshold)
}
return err
}

316
internal/router/m2_test.go Normal file
View file

@ -0,0 +1,316 @@
package router
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"codeberg.org/squidd/parsimony/internal/outbound"
"codeberg.org/squidd/parsimony/internal/store"
)
type env struct {
r *Router
s *store.Store
out *outbound.Fake
list store.List
ctx context.Context
alice int64
}
func newEnv(t *testing.T) *env {
t.Helper()
s, err := store.Open(":memory:")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
l, _ := s.CreateList("dev", "example.org", "Dev")
s.AddMember(l.ID, "alice@example.com", "Alice")
s.AddMember(l.ID, "bob@example.com", "Bob")
ms, _ := s.Members(l.ID)
return &env{r: New(s), s: s, out: &outbound.Fake{}, list: l, ctx: context.Background(), alice: ms[0].ID}
}
func lastMail(t *testing.T, out *outbound.Fake) outbound.Sent {
t.Helper()
if len(out.Sent) == 0 {
t.Fatal("no mail was sent")
}
return out.Sent[len(out.Sent)-1]
}
func TestSendUsesVERPPerRecipient(t *testing.T) {
e := newEnv(t)
res, err := e.r.Route("alice@example.com", "dev@example.org", []byte(sample))
if err != nil {
t.Fatal(err)
}
if err := e.r.Send(e.ctx, e.out, res); err != nil {
t.Fatal(err)
}
if len(e.out.Sent) != 2 {
t.Fatalf("want one send per recipient, got %d", len(e.out.Sent))
}
seen := map[string]bool{}
for _, m := range e.out.Sent {
if len(m.To) != 1 {
t.Fatalf("to = %v", m.To)
}
if !strings.HasPrefix(m.From, "dev+bounce-") || !strings.HasSuffix(m.From, "@example.org") {
t.Fatalf("envelope from = %q", m.From)
}
seen[m.From] = true
}
if len(seen) != 2 {
t.Fatal("each recipient needs a distinct bounce address")
}
}
func TestDisabledMembersAreSkipped(t *testing.T) {
e := newEnv(t)
for i := 0; i < 3; i++ {
e.s.RecordHardBounce(e.alice, 3)
}
res, err := e.r.Route("bob@example.com", "dev@example.org", []byte(sample))
if err != nil {
t.Fatal(err)
}
if len(res.Recipients) != 1 || res.Recipients[0].Address != "bob@example.com" {
t.Fatalf("recipients: %+v", res.Recipients)
}
}
func TestModeratedListHoldsPosts(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen)
for _, sender := range []string{"alice@example.com", "stranger@x.com"} {
_, err := e.r.Route(sender, "dev@example.org", []byte(sample))
if !errors.Is(err, ErrHeld) {
t.Fatalf("%s: want ErrHeld, got %v", sender, err)
}
}
held, _ := e.s.ListHeld(e.list.ID)
if len(held) != 2 {
t.Fatalf("held: %d", len(held))
}
if len(e.out.Sent) != 0 {
t.Fatal("nothing may be sent while held")
}
}
func TestReleaseHeld(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen)
e.r.Route("stranger@x.com", "dev@example.org", []byte(sample))
held, _ := e.s.ListHeld(e.list.ID)
if err := e.r.Release(e.ctx, e.out, held[0].ID); err != nil {
t.Fatal(err)
}
if len(e.out.Sent) != 2 {
t.Fatalf("sent = %d", len(e.out.Sent))
}
if !strings.Contains(string(e.out.Sent[0].Data), "List-Id: Dev <dev.example.org>") {
t.Error("released mail must carry list headers")
}
if rest, _ := e.s.ListHeld(e.list.ID); len(rest) != 0 {
t.Fatal("released message must leave the queue")
}
if err := e.r.Release(e.ctx, e.out, held[0].ID); !errors.Is(err, store.ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func token(t *testing.T, m outbound.Sent) string {
t.Helper()
const marker = "Reply-To: dev+confirm-"
i := strings.Index(string(m.Data), marker)
if i < 0 {
t.Fatalf("no confirm address in %q", m.Data)
}
rest := string(m.Data)[i+len(marker):]
return rest[:strings.Index(rest, "@")]
}
func TestJoinFlow(t *testing.T) {
e := newEnv(t)
handled, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
if !handled || err != nil {
t.Fatalf("join: %v %v", handled, err)
}
m := lastMail(t, e.out)
if len(m.To) != 1 || m.To[0] != "carol@x.com" {
t.Fatalf("confirmation must go to the requester: %+v", m.To)
}
if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok {
t.Fatal("not a member before confirming")
}
tok := token(t, m)
handled, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
if !handled || err != nil {
t.Fatalf("confirm: %v %v", handled, err)
}
if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok {
t.Fatal("carol must be a member after confirming")
}
_, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
if !errors.Is(err, ErrBadToken) {
t.Fatalf("reused token: want ErrBadToken, got %v", err)
}
}
func TestJoinRateLimitedAndSilentForMembers(t *testing.T) {
e := newEnv(t)
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
if len(e.out.Sent) != 1 {
t.Fatalf("second request must not send mail, sent %d", len(e.out.Sent))
}
e.r.Command(e.ctx, e.out, "alice@example.com", "dev+join@example.org", nil)
if len(e.out.Sent) != 1 {
t.Fatal("existing members get no mail")
}
e.r.Command(e.ctx, e.out, "", "dev+join@example.org", nil)
if len(e.out.Sent) != 1 {
t.Fatal("empty sender is ignored")
}
}
func TestJoinClosed(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed)
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
if !errors.Is(err, ErrJoinClosed) {
t.Fatalf("want ErrJoinClosed, got %v", err)
}
if len(e.out.Sent) != 0 {
t.Fatal("no mail for closed lists")
}
}
func TestConfirmAfterListClosed(t *testing.T) {
e := newEnv(t)
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
tok := token(t, lastMail(t, e.out))
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed)
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
if !errors.Is(err, ErrJoinClosed) {
t.Fatalf("want ErrJoinClosed, got %v", err)
}
if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok {
t.Fatal("must not join a closed list")
}
}
func TestLeaveFlow(t *testing.T) {
e := newEnv(t)
e.r.Command(e.ctx, e.out, "alice@example.com", "dev+leave@example.org", nil)
m := lastMail(t, e.out)
if m.To[0] != "alice@example.com" {
t.Fatalf("to = %v", m.To)
}
if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); !ok {
t.Fatal("still a member before confirming")
}
e.r.Command(e.ctx, e.out, "alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil)
if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); ok {
t.Fatal("alice must be gone")
}
// Leaving when not a member sends nothing.
n := len(e.out.Sent)
e.r.Command(e.ctx, e.out, "zed@x.com", "dev+leave@example.org", nil)
if len(e.out.Sent) != n {
t.Fatal("non-members get no mail")
}
}
func TestConfirmTokenFromOtherList(t *testing.T) {
e := newEnv(t)
e.s.CreateList("ops", "example.org", "Ops")
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
tok := token(t, lastMail(t, e.out))
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "ops+confirm-"+tok+"@example.org", nil)
if !errors.Is(err, ErrBadToken) {
t.Fatalf("want ErrBadToken, got %v", err)
}
}
func TestCommandNotHandledForPlainAddress(t *testing.T) {
e := newEnv(t)
handled, err := e.r.Command(e.ctx, e.out, "a@x.com", "dev@example.org", nil)
if handled || err != nil {
t.Fatalf("plain address: %v %v", handled, err)
}
}
func TestUnknownCommandAndList(t *testing.T) {
e := newEnv(t)
for _, rcpt := range []string{"dev+bogus@example.org", "nope+join@example.org"} {
handled, err := e.r.Command(e.ctx, e.out, "a@x.com", rcpt, nil)
if !handled || !errors.Is(err, ErrUnknownList) {
t.Errorf("%s: %v %v", rcpt, handled, err)
}
}
}
func bounceMsg(status string) []byte {
return []byte(fmt.Sprintf("Subject: Undelivered Mail\r\nContent-Type: text/plain\r\n\r\nAction: failed\r\nStatus: %s\r\nDiagnostic-Code: smtp; 550 no such user\r\n", status))
}
func TestHardBouncesDisableMember(t *testing.T) {
e := newEnv(t)
addr := e.r.bounceAddress(e.list, e.alice)
for i := 0; i < 3; i++ {
handled, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1"))
if !handled || err != nil {
t.Fatalf("bounce: %v %v", handled, err)
}
}
m, _ := e.s.MemberByID(e.alice)
if !m.Disabled || m.BounceCount != 3 {
t.Fatalf("member: %+v", m)
}
}
func TestSoftBounceIgnored(t *testing.T) {
e := newEnv(t)
e.r.Command(e.ctx, e.out, "", e.r.bounceAddress(e.list, e.alice), bounceMsg("4.2.2"))
if m, _ := e.s.MemberByID(e.alice); m.BounceCount != 0 {
t.Fatalf("soft bounce counted: %+v", m)
}
}
func TestForgedBounceRejected(t *testing.T) {
e := newEnv(t)
addr := fmt.Sprintf("dev+bounce-%d-000000000000@example.org", e.alice)
_, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1"))
if !errors.Is(err, ErrBadToken) {
t.Fatalf("want ErrBadToken, got %v", err)
}
if m, _ := e.s.MemberByID(e.alice); m.BounceCount != 0 {
t.Fatal("forged bounce must not count")
}
}
func TestBounceFromOtherListRejected(t *testing.T) {
e := newEnv(t)
e.s.CreateList("ops", "example.org", "Ops")
ops, _ := e.s.FindList("ops@example.org")
addr := e.r.bounceAddress(ops, e.alice) // valid MAC but alice is not in ops
_, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1"))
if !errors.Is(err, ErrBadToken) {
t.Fatalf("want ErrBadToken, got %v", err)
}
}
func TestSystemBounceAddressDiscarded(t *testing.T) {
e := newEnv(t)
handled, err := e.r.Command(e.ctx, e.out, "", "dev+bounce@example.org", bounceMsg("5.1.1"))
if !handled || err != nil {
t.Fatalf("%v %v", handled, err)
}
}

View file

@ -2,23 +2,40 @@
package router
import (
"context"
"errors"
"fmt"
"log"
"sync"
"codeberg.org/squidd/parsimony/internal/outbound"
"codeberg.org/squidd/parsimony/internal/store"
)
var (
// ErrUnknownList means the recipient address is not a list.
// ErrUnknownList means the recipient address is not a list or a list command.
ErrUnknownList = errors.New("unknown list")
// ErrNotMember means the sender may not post to the list.
ErrNotMember = errors.New("sender is not a member")
// ErrHeld means the message was queued for moderation. It is not a failure.
ErrHeld = errors.New("held for moderation")
// ErrBadToken means a confirmation or bounce token is invalid, used or expired.
ErrBadToken = errors.New("invalid or expired token")
// ErrJoinClosed means the list does not accept self-service subscriptions.
ErrJoinClosed = errors.New("list is closed")
)
// Recipient is a subscriber that receives a message.
type Recipient struct {
ID int64 // member id, used for the VERP bounce address
Address string
}
// Result describes how to deliver an accepted message.
type Result struct {
List store.List
Recipients []string // subscriber addresses
Data []byte // message to send
Recipients []Recipient
Data []byte // message to send, with list headers
}
// Router applies list policy to incoming messages.
@ -26,12 +43,17 @@ type Router struct {
store *store.Store
// BaseURL is the public web URL; enables one-click unsubscribe when set.
BaseURL string
secretOnce sync.Once
secretVal []byte
secretErr error
}
// New creates a Router backed by s.
func New(s *store.Store) *Router { return &Router{store: s} }
// Route validates a message from sender to the list address rcpt.
// Moderated lists queue the message and return ErrHeld.
func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) {
l, err := r.store.FindList(rcpt)
if errors.Is(err, store.ErrNotFound) {
@ -40,6 +62,12 @@ func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) {
if err != nil {
return Result{}, err
}
if l.PostPolicy == store.PostModerated {
if _, err := r.store.HoldMessage(l.ID, sender, data); err != nil {
return Result{}, err
}
return Result{}, ErrHeld
}
ok, err := r.store.IsMember(l.ID, sender)
if err != nil {
return Result{}, err
@ -47,13 +75,18 @@ func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) {
if !ok {
return Result{}, ErrNotMember
}
members, err := r.store.Members(l.ID)
return r.prepare(l, data)
}
// prepare builds the delivery plan for a list.
func (r *Router) prepare(l store.List, data []byte) (Result, error) {
members, err := r.store.ActiveMembers(l.ID)
if err != nil {
return Result{}, err
}
rcpts := make([]string, len(members))
rcpts := make([]Recipient, len(members))
for i, m := range members {
rcpts[i] = m.Address
rcpts[i] = Recipient{ID: m.ID, Address: m.Address}
}
out, err := rewrite(l, r.BaseURL, data)
if err != nil {
@ -61,3 +94,44 @@ func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) {
}
return Result{List: l, Recipients: rcpts, Data: out}, nil
}
// Send delivers the message to every recipient, each with its own signed
// bounce address (VERP). Individual failures are logged and skipped; an error is
// returned only when nobody could be reached, so a retry cannot duplicate mail.
func (r *Router) Send(ctx context.Context, out outbound.Sender, res Result) error {
failed := 0
for _, rc := range res.Recipients {
from := r.bounceAddress(res.List, rc.ID)
if from == "" {
return errors.New("cannot create bounce address")
}
if err := out.Send(ctx, from, []string{rc.Address}, res.Data); err != nil {
failed++
log.Printf("deliver %s to %s: %v", res.List.Address(), rc.Address, err)
}
}
if failed > 0 && failed == len(res.Recipients) {
return fmt.Errorf("delivery failed for all %d recipients", failed)
}
return nil
}
// Release approves a held message and delivers it.
func (r *Router) Release(ctx context.Context, out outbound.Sender, heldID int64) error {
h, err := r.store.GetHeld(heldID)
if err != nil {
return err
}
l, err := r.store.GetList(h.ListID)
if err != nil {
return err
}
res, err := r.prepare(l, h.Data)
if err != nil {
return err
}
if err := r.Send(ctx, out, res); err != nil {
return err
}
return r.store.DeleteHeld(heldID)
}

185
internal/store/m2_test.go Normal file
View file

@ -0,0 +1,185 @@
package store
import (
"database/sql"
"errors"
"path/filepath"
"testing"
"time"
)
func TestListPolicies(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
if l.PostPolicy != "members" || l.JoinPolicy != "open" {
t.Fatalf("defaults: %+v", l)
}
if err := s.SetPolicies(l.ID, "moderated", "closed"); err != nil {
t.Fatal(err)
}
got, _ := s.GetList(l.ID)
if got.PostPolicy != "moderated" || got.JoinPolicy != "closed" {
t.Fatalf("policies: %+v", got)
}
if err := s.SetPolicies(l.ID, "bogus", "open"); err == nil {
t.Fatal("invalid policy must be rejected")
}
if err := s.SetPolicies(999, "members", "open"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestHeldQueue(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
id, err := s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: hi\r\n\r\nx"))
if err != nil {
t.Fatal(err)
}
hs, err := s.ListHeld(l.ID)
if err != nil || len(hs) != 1 || hs[0].ID != id || hs[0].Sender != "bob@x.com" {
t.Fatalf("held: %+v %v", hs, err)
}
h, err := s.GetHeld(id)
if err != nil || string(h.Data) != "Subject: hi\r\n\r\nx" || h.ListID != l.ID {
t.Fatalf("get: %+v %v", h, err)
}
if err := s.DeleteHeld(id); err != nil {
t.Fatal(err)
}
if _, err := s.GetHeld(id); !errors.Is(err, ErrNotFound) {
t.Fatal("held message should be gone")
}
if err := s.DeleteHeld(id); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestPendingTokens(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
l, _ := s.CreateList("dev", "example.org", "Dev")
tok, created, err := s.CreatePending(l.ID, "A@x.com", "join")
if err != nil || !created || tok == "" {
t.Fatalf("create: %q %v %v", tok, created, err)
}
// Rate limit: same request within an hour is not recreated.
now = now.Add(30 * time.Minute)
if _, created, _ := s.CreatePending(l.ID, "a@x.com", "join"); created {
t.Fatal("must be rate limited")
}
// A different action is independent.
if _, created, _ := s.CreatePending(l.ID, "a@x.com", "leave"); !created {
t.Fatal("leave is a separate request")
}
p, err := s.ConfirmPending(tok)
if err != nil || p.Address != "a@x.com" || p.Action != "join" || p.ListID != l.ID {
t.Fatalf("confirm: %+v %v", p, err)
}
if _, err := s.ConfirmPending(tok); !errors.Is(err, ErrNotFound) {
t.Fatal("token is single use")
}
if _, err := s.ConfirmPending("nope"); !errors.Is(err, ErrNotFound) {
t.Fatal("unknown token")
}
// Expiry.
tok2, _, _ := s.CreatePending(l.ID, "b@x.com", "join")
now = now.Add(73 * time.Hour)
if _, err := s.ConfirmPending(tok2); !errors.Is(err, ErrNotFound) {
t.Fatal("token must expire after 72h")
}
}
func TestSecretIsStable(t *testing.T) {
s := newTestStore(t)
a, err := s.Secret()
if err != nil || len(a) < 32 {
t.Fatalf("secret: %x %v", a, err)
}
b, _ := s.Secret()
if string(a) != string(b) {
t.Fatal("secret must not change")
}
}
func TestBounceDisablesAfterThreshold(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.AddMember(l.ID, "a@x.com", "A")
ms, _ := s.Members(l.ID)
id := ms[0].ID
for i := 1; i <= 2; i++ {
disabled, err := s.RecordHardBounce(id, 3)
if err != nil || disabled {
t.Fatalf("bounce %d: %v %v", i, disabled, err)
}
}
disabled, _ := s.RecordHardBounce(id, 3)
if !disabled {
t.Fatal("third hard bounce must disable")
}
ms, _ = s.Members(l.ID)
if !ms[0].Disabled || ms[0].BounceCount != 3 {
t.Fatalf("member: %+v", ms[0])
}
if act, _ := s.ActiveMembers(l.ID); len(act) != 0 {
t.Fatalf("active: %+v", act)
}
// Re-adding resets the state.
s.AddMember(l.ID, "a@x.com", "A")
act, _ := s.ActiveMembers(l.ID)
if len(act) != 1 || act[0].BounceCount != 0 || act[0].ID != id {
t.Fatalf("after re-add: %+v", act)
}
m, err := s.MemberByID(id)
if err != nil || m.Address != "a@x.com" || m.ListID != l.ID {
t.Fatalf("by id: %+v %v", m, err)
}
if _, err := s.MemberByID(999); !errors.Is(err, ErrNotFound) {
t.Fatal("unknown id")
}
if _, err := s.RecordHardBounce(999, 3); !errors.Is(err, ErrNotFound) {
t.Fatal("unknown id")
}
}
func TestMigrateFromM1(t *testing.T) {
path := filepath.Join(t.TempDir(), "old.db")
db, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
_, err = db.Exec(`
CREATE TABLE lists (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, domain TEXT NOT NULL, title TEXT NOT NULL DEFAULT '', UNIQUE(name, domain));
CREATE TABLE members (list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, address TEXT NOT NULL, display_name TEXT NOT NULL DEFAULT '', PRIMARY KEY(list_id, address));
INSERT INTO lists(name, domain, title) VALUES('dev','example.org','Dev');
INSERT INTO members(list_id, address, display_name) VALUES(1,'a@x.com','A'),(1,'b@x.com','B');`)
if err != nil {
t.Fatal(err)
}
db.Close()
s, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s.Close()
l, err := s.FindList("dev@example.org")
if err != nil || l.PostPolicy != "members" || l.JoinPolicy != "open" {
t.Fatalf("list: %+v %v", l, err)
}
ms, _ := s.Members(l.ID)
if len(ms) != 2 || ms[0].ID == 0 || ms[0].Address != "a@x.com" {
t.Fatalf("members: %+v", ms)
}
s.Close()
// Opening again is a no-op.
s2, err := Open(path)
if err != nil {
t.Fatal(err)
}
s2.Close()
}

View file

@ -1,27 +1,52 @@
// Package store persists lists and members in SQLite.
// Package store persists lists, members and moderation state in SQLite.
package store
import (
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
_ "modernc.org/sqlite"
)
// ErrNotFound is returned when a list or member does not exist.
var ErrNotFound = errors.New("not found")
var (
// ErrNotFound is returned when a record does not exist.
ErrNotFound = errors.New("not found")
// ErrExists is returned when creating something that already exists.
ErrExists = errors.New("already exists")
)
// ErrExists is returned when creating something that already exists.
var ErrExists = errors.New("already exists")
// Policy values.
const (
PostMembers = "members" // only members may post, others are rejected
PostModerated = "moderated" // every post waits for approval
JoinOpen = "open" // anyone can subscribe (with confirmation)
JoinClosed = "closed" // only the admin adds members
)
// Pending actions.
const (
ActionJoin = "join"
ActionLeave = "leave"
)
const (
pendingTTL = 72 * time.Hour
pendingRateLimit = time.Hour
)
// List is a mailing list.
type List struct {
ID int64
Name string // local part, lowercase
Domain string // lowercase
Title string
ID int64
Name string // local part, lowercase
Domain string // lowercase
Title string
PostPolicy string
JoinPolicy string
}
// Address returns the list posting address.
@ -29,26 +54,74 @@ func (l List) Address() string { return l.Name + "@" + l.Domain }
// Member is a list subscriber.
type Member struct {
ID int64
ListID int64
Address string // lowercase
DisplayName string
Disabled bool // too many hard bounces: no more delivery
BounceCount int
}
// Held is a message waiting for moderation.
type Held struct {
ID int64
ListID int64
Sender string
Data []byte
CreatedAt time.Time
}
// Pending is an unconfirmed subscribe or unsubscribe request.
type Pending struct {
Token string
ListID int64
Address string
Action string
}
// Store wraps the SQLite database.
type Store struct{ db *sql.DB }
type Store struct {
db *sql.DB
// Now returns the current time; tests replace it.
Now func() time.Time
}
const schema = `
CREATE TABLE IF NOT EXISTS lists (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
domain TEXT NOT NULL,
title TEXT NOT NULL DEFAULT '',
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
domain TEXT NOT NULL,
title TEXT NOT NULL DEFAULT '',
post_policy TEXT NOT NULL DEFAULT 'members',
join_policy TEXT NOT NULL DEFAULT 'open',
UNIQUE(name, domain)
);
CREATE TABLE IF NOT EXISTS members (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
address TEXT NOT NULL,
display_name TEXT NOT NULL DEFAULT '',
PRIMARY KEY(list_id, address)
disabled INTEGER NOT NULL DEFAULT 0,
bounce_count INTEGER NOT NULL DEFAULT 0,
UNIQUE(list_id, address)
);
CREATE TABLE IF NOT EXISTS held (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
sender TEXT NOT NULL,
data BLOB NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS pending (
token TEXT PRIMARY KEY,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
address TEXT NOT NULL,
action TEXT NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);`
// Open opens (and migrates) the database at path. Use ":memory:" for tests.
@ -63,106 +136,60 @@ func Open(path string) (*Store, error) {
db.Close()
return nil, err
}
if _, err := db.Exec(schema); err != nil {
if err := migrate(db); err != nil {
db.Close()
return nil, fmt.Errorf("migrate: %w", err)
}
return &Store{db: db}, nil
return &Store{db: db, Now: time.Now}, nil
}
func hasColumn(db *sql.DB, table, column string) (bool, error) {
var n int
err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info(?) WHERE name=?`, table, column).Scan(&n)
return n > 0, err
}
// migrate upgrades a database created by milestone 1, then ensures the schema.
func migrate(db *sql.DB) error {
if ok, err := hasColumn(db, "members", "list_id"); err != nil {
return err
} else if ok {
if hasID, err := hasColumn(db, "members", "id"); err != nil {
return err
} else if !hasID {
if _, err := db.Exec(`ALTER TABLE members RENAME TO members_v1`); err != nil {
return err
}
if _, err := db.Exec(schema); err != nil {
return err
}
if _, err := db.Exec(`INSERT INTO members(list_id, address, display_name)
SELECT list_id, address, display_name FROM members_v1 ORDER BY list_id, address;
DROP TABLE members_v1;`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "lists", "name"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "lists", "post_policy"); !has {
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN post_policy TEXT NOT NULL DEFAULT 'members';
ALTER TABLE lists ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'open';`); err != nil {
return err
}
}
}
_, err := db.Exec(schema)
return err
}
// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }
// CreateList adds a new list.
func (s *Store) CreateList(name, domain, title string) (List, error) {
l := List{Name: strings.ToLower(name), Domain: strings.ToLower(domain), Title: title}
res, err := s.db.Exec(`INSERT INTO lists(name, domain, title) VALUES(?,?,?)`, l.Name, l.Domain, l.Title)
if err != nil {
if strings.Contains(err.Error(), "UNIQUE") {
return List{}, ErrExists
}
return List{}, err
}
l.ID, _ = res.LastInsertId()
return l, nil
}
func norm(addr string) string { return strings.ToLower(strings.TrimSpace(addr)) }
// FindList resolves a posting address like "dev@example.org".
func (s *Store) FindList(addr string) (List, error) {
name, domain, ok := strings.Cut(strings.ToLower(strings.TrimSpace(addr)), "@")
if !ok {
return List{}, ErrNotFound
}
var l List
err := s.db.QueryRow(`SELECT id, name, domain, title FROM lists WHERE name=? AND domain=?`, name, domain).
Scan(&l.ID, &l.Name, &l.Domain, &l.Title)
if errors.Is(err, sql.ErrNoRows) {
return List{}, ErrNotFound
}
return l, err
}
// AddMember subscribes an address to a list (idempotent).
func (s *Store) AddMember(listID int64, addr, displayName string) error {
_, err := s.db.Exec(`INSERT OR REPLACE INTO members(list_id, address, display_name) VALUES(?,?,?)`,
listID, strings.ToLower(strings.TrimSpace(addr)), displayName)
return err
}
// IsMember reports whether addr is subscribed to the list.
func (s *Store) IsMember(listID int64, addr string) (bool, error) {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=?`,
listID, strings.ToLower(strings.TrimSpace(addr))).Scan(&n)
return n > 0, err
}
// Members returns all subscribers of a list.
func (s *Store) Members(listID int64) ([]Member, error) {
rows, err := s.db.Query(`SELECT address, display_name FROM members WHERE list_id=? ORDER BY address`, listID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Member
for rows.Next() {
var m Member
if err := rows.Scan(&m.Address, &m.DisplayName); err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// ListLists returns every list ordered by address.
func (s *Store) ListLists() ([]List, error) {
rows, err := s.db.Query(`SELECT id, name, domain, title FROM lists ORDER BY domain, name`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []List
for rows.Next() {
var l List
if err := rows.Scan(&l.ID, &l.Name, &l.Domain, &l.Title); err != nil {
return nil, err
}
out = append(out, l)
}
return out, rows.Err()
}
// GetList returns a list by id.
func (s *Store) GetList(id int64) (List, error) {
var l List
err := s.db.QueryRow(`SELECT id, name, domain, title FROM lists WHERE id=?`, id).
Scan(&l.ID, &l.Name, &l.Domain, &l.Title)
if errors.Is(err, sql.ErrNoRows) {
return List{}, ErrNotFound
}
return l, err
}
func isUnique(err error) bool { return err != nil && strings.Contains(err.Error(), "UNIQUE") }
func affected(res sql.Result, err error) error {
if err != nil {
@ -174,18 +201,287 @@ func affected(res sql.Result, err error) error {
return nil
}
// Lists.
const listCols = `id, name, domain, title, post_policy, join_policy`
func scanList(row interface{ Scan(...any) error }) (List, error) {
var l List
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy)
if errors.Is(err, sql.ErrNoRows) {
return List{}, ErrNotFound
}
return l, err
}
// CreateList adds a new list with default policies.
func (s *Store) CreateList(name, domain, title string) (List, error) {
res, err := s.db.Exec(`INSERT INTO lists(name, domain, title) VALUES(?,?,?)`,
strings.ToLower(name), strings.ToLower(domain), title)
if isUnique(err) {
return List{}, ErrExists
}
if err != nil {
return List{}, err
}
id, _ := res.LastInsertId()
return s.GetList(id)
}
// FindList resolves a posting address like "dev@example.org".
func (s *Store) FindList(addr string) (List, error) {
name, domain, ok := strings.Cut(norm(addr), "@")
if !ok {
return List{}, ErrNotFound
}
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE name=? AND domain=?`, name, domain))
}
// GetList returns a list by id.
func (s *Store) GetList(id int64) (List, error) {
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE id=?`, id))
}
// ListLists returns every list ordered by address.
func (s *Store) ListLists() ([]List, error) {
rows, err := s.db.Query(`SELECT ` + listCols + ` FROM lists ORDER BY domain, name`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []List
for rows.Next() {
l, err := scanList(rows)
if err != nil {
return nil, err
}
out = append(out, l)
}
return out, rows.Err()
}
// UpdateList changes the title of a list.
func (s *Store) UpdateList(id int64, title string) error {
return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id))
}
// DeleteList removes a list and its members.
// SetPolicies changes the post and join policies of a list.
func (s *Store) SetPolicies(id int64, post, join string) error {
if post != PostMembers && post != PostModerated {
return fmt.Errorf("invalid post policy %q", post)
}
if join != JoinOpen && join != JoinClosed {
return fmt.Errorf("invalid join policy %q", join)
}
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=? WHERE id=?`, post, join, id))
}
// DeleteList removes a list with its members, held messages and pending requests.
func (s *Store) DeleteList(id int64) error {
return affected(s.db.Exec(`DELETE FROM lists WHERE id=?`, id))
}
// Members.
const memberCols = `id, list_id, address, display_name, disabled, bounce_count`
func scanMember(row interface{ Scan(...any) error }) (Member, error) {
var m Member
var dis int
err := row.Scan(&m.ID, &m.ListID, &m.Address, &m.DisplayName, &dis, &m.BounceCount)
if errors.Is(err, sql.ErrNoRows) {
return Member{}, ErrNotFound
}
m.Disabled = dis != 0
return m, err
}
// AddMember subscribes an address to a list. Re-adding keeps the member id and
// clears the bounce state.
func (s *Store) AddMember(listID int64, addr, displayName string) error {
_, err := s.db.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?)
ON CONFLICT(list_id, address) DO UPDATE SET display_name=excluded.display_name, disabled=0, bounce_count=0`,
listID, norm(addr), displayName)
return err
}
// IsMember reports whether addr is subscribed to the list.
func (s *Store) IsMember(listID int64, addr string) (bool, error) {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=?`, listID, norm(addr)).Scan(&n)
return n > 0, err
}
func (s *Store) queryMembers(where string, args ...any) ([]Member, error) {
rows, err := s.db.Query(`SELECT `+memberCols+` FROM members WHERE `+where+` ORDER BY address`, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Member
for rows.Next() {
m, err := scanMember(rows)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// Members returns all subscribers of a list, including disabled ones.
func (s *Store) Members(listID int64) ([]Member, error) {
return s.queryMembers(`list_id=?`, listID)
}
// ActiveMembers returns the subscribers that should receive mail.
func (s *Store) ActiveMembers(listID int64) ([]Member, error) {
return s.queryMembers(`list_id=? AND disabled=0`, listID)
}
// MemberByID returns a member by its id.
func (s *Store) MemberByID(id int64) (Member, error) {
return scanMember(s.db.QueryRow(`SELECT `+memberCols+` FROM members WHERE id=?`, id))
}
// RemoveMember unsubscribes an address from a list.
func (s *Store) RemoveMember(listID int64, addr string) error {
return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`,
listID, strings.ToLower(strings.TrimSpace(addr))))
return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr)))
}
// RecordHardBounce counts a hard bounce and disables the member once the count
// reaches threshold. It reports whether the member is now disabled.
func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) {
if err := affected(s.db.Exec(`UPDATE members SET bounce_count=bounce_count+1 WHERE id=?`, memberID)); err != nil {
return false, err
}
if _, err := s.db.Exec(`UPDATE members SET disabled=1 WHERE id=? AND bounce_count>=?`, memberID, threshold); err != nil {
return false, err
}
m, err := s.MemberByID(memberID)
return m.Disabled, err
}
// Held messages.
// HoldMessage queues a message for moderation and returns its id.
func (s *Store) HoldMessage(listID int64, sender string, data []byte) (int64, error) {
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at) VALUES(?,?,?,?)`,
listID, norm(sender), data, s.Now().Unix())
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func scanHeld(row interface{ Scan(...any) error }) (Held, error) {
var h Held
var ts int64
err := row.Scan(&h.ID, &h.ListID, &h.Sender, &h.Data, &ts)
if errors.Is(err, sql.ErrNoRows) {
return Held{}, ErrNotFound
}
h.CreatedAt = time.Unix(ts, 0)
return h, err
}
// ListHeld returns the held messages of a list, oldest first.
func (s *Store) ListHeld(listID int64) ([]Held, error) {
rows, err := s.db.Query(`SELECT id, list_id, sender, data, created_at FROM held WHERE list_id=? ORDER BY id`, listID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Held
for rows.Next() {
h, err := scanHeld(rows)
if err != nil {
return nil, err
}
out = append(out, h)
}
return out, rows.Err()
}
// GetHeld returns a held message by id.
func (s *Store) GetHeld(id int64) (Held, error) {
return scanHeld(s.db.QueryRow(`SELECT id, list_id, sender, data, created_at FROM held WHERE id=?`, id))
}
// DeleteHeld discards a held message.
func (s *Store) DeleteHeld(id int64) error {
return affected(s.db.Exec(`DELETE FROM held WHERE id=?`, id))
}
// Pending confirmations.
func randomHex(n int) (string, error) {
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// CreatePending records a subscribe or unsubscribe request and returns its
// token. created is false when an identical request was made in the last hour:
// this limits how often a third party can trigger confirmation mail.
func (s *Store) CreatePending(listID int64, addr, action string) (token string, created bool, err error) {
addr = norm(addr)
now := s.Now()
var n int
err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE list_id=? AND address=? AND action=? AND created_at>?`,
listID, addr, action, now.Add(-pendingRateLimit).Unix()).Scan(&n)
if err != nil || n > 0 {
return "", false, err
}
token, err = randomHex(16)
if err != nil {
return "", false, err
}
_, err = s.db.Exec(`INSERT INTO pending(token, list_id, address, action, created_at) VALUES(?,?,?,?,?)`,
token, listID, addr, action, now.Unix())
if err != nil {
return "", false, err
}
return token, true, nil
}
// ConfirmPending consumes a token. Unknown, used and expired tokens give ErrNotFound.
func (s *Store) ConfirmPending(token string) (Pending, error) {
var p Pending
var ts int64
err := s.db.QueryRow(`SELECT token, list_id, address, action, created_at FROM pending WHERE token=?`, token).
Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts)
if errors.Is(err, sql.ErrNoRows) {
return Pending{}, ErrNotFound
}
if err != nil {
return Pending{}, err
}
if _, err := s.db.Exec(`DELETE FROM pending WHERE token=?`, token); err != nil {
return Pending{}, err
}
if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL {
return Pending{}, ErrNotFound
}
return p, nil
}
// Secret returns the HMAC secret, generating and saving it on first use.
func (s *Store) Secret() ([]byte, error) {
var v string
err := s.db.QueryRow(`SELECT value FROM settings WHERE key='secret'`).Scan(&v)
if errors.Is(err, sql.ErrNoRows) {
v, err = randomHex(32)
if err != nil {
return nil, err
}
if _, err = s.db.Exec(`INSERT INTO settings(key, value) VALUES('secret', ?)`, v); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return hex.DecodeString(v)
}