From 28b73fa24a8226e5d3207d8894d5e489a256c408 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:49:32 +0100 Subject: [PATCH 1/9] test(api): every route but health answers 401 without the token routes() is now the single table New registers, so a route added later is covered by TestEveryRouteNeedsTheToken without editing it. - api: declare the routes in one table (routes) and register from it - api: table test walks it, 401 without a token, not 401 with one --- internal/api/api.go | 55 +++++++++++++++++++++++++--------------- internal/api/api_test.go | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 20 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index b40b9ab..e9e7200 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -40,29 +40,44 @@ func New(s *store.Store, token string, rel Releaser) http.Handler { func newHandler(s *store.Store, token string, rel Releaser, now func() time.Time) http.Handler { srv := &server{store: s, token: token, releaser: rel, lim: newLimiter(now)} mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/health", func(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) - }) - mux.Handle("GET /api/v1/lists", srv.auth(srv.listLists)) - mux.Handle("POST /api/v1/lists", srv.auth(srv.createList)) - mux.Handle("GET /api/v1/lists/{id}", srv.auth(srv.getList)) - mux.Handle("PATCH /api/v1/lists/{id}", srv.auth(srv.updateList)) - mux.Handle("DELETE /api/v1/lists/{id}", srv.auth(srv.deleteList)) - mux.Handle("GET /api/v1/lists/{id}/members", srv.auth(srv.listMembers)) - mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember)) - mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember)) - mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive)) - mux.Handle("POST /api/v1/lists/{id}/archive/import", srv.auth(srv.importArchive)) - mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived)) - mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived)) - mux.Handle("GET /api/v1/queue", srv.auth(srv.getQueue)) - mux.Handle("DELETE /api/v1/queue/{id}", srv.auth(srv.discardQueued)) - mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld)) - mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld)) - mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld)) + for _, rt := range srv.routes() { + mux.Handle(rt.pattern, rt.handler) + } return mux } +type route struct { + pattern string + handler http.Handler +} + +// routes is the only place routes are declared. Everything but the health check goes through auth. +func (s *server) routes() []route { + public := func(h http.HandlerFunc) http.Handler { return h } + return []route{ + {"GET /api/v1/health", public(func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + })}, + {"GET /api/v1/lists", s.auth(s.listLists)}, + {"POST /api/v1/lists", s.auth(s.createList)}, + {"GET /api/v1/lists/{id}", s.auth(s.getList)}, + {"PATCH /api/v1/lists/{id}", s.auth(s.updateList)}, + {"DELETE /api/v1/lists/{id}", s.auth(s.deleteList)}, + {"GET /api/v1/lists/{id}/members", s.auth(s.listMembers)}, + {"POST /api/v1/lists/{id}/members", s.auth(s.addMember)}, + {"DELETE /api/v1/lists/{id}/members/{address}", s.auth(s.removeMember)}, + {"GET /api/v1/lists/{id}/archive", s.auth(s.listArchive)}, + {"POST /api/v1/lists/{id}/archive/import", s.auth(s.importArchive)}, + {"GET /api/v1/archive/{id}", s.auth(s.getArchived)}, + {"DELETE /api/v1/archive/{id}", s.auth(s.deleteArchived)}, + {"GET /api/v1/queue", s.auth(s.getQueue)}, + {"DELETE /api/v1/queue/{id}", s.auth(s.discardQueued)}, + {"GET /api/v1/lists/{id}/held", s.auth(s.listHeld)}, + {"POST /api/v1/held/{id}/approve", s.auth(s.approveHeld)}, + {"DELETE /api/v1/held/{id}", s.auth(s.discardHeld)}, + } +} + func (s *server) auth(next http.HandlerFunc) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ip := clientIP(r) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index ac1e3ad..984dda7 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -685,3 +685,50 @@ func TestQueueShowsFailedDeliveriesApartFromPendingOnes(t *testing.T) { } expect(t, do(t, h, "DELETE", "/api/v1/queue/"+strconv.FormatInt(due[2].ID, 10), "", true), 204) } + +// TestEveryRouteNeedsTheToken walks the route table New registers, so a route added later is +// covered without touching this test: all of them but the health check answer 401 without a token +// and something else with one. +func TestEveryRouteNeedsTheToken(t *testing.T) { + s, err := store.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + h := New(s, token, nil) + + rs := (&server{}).routes() + if len(rs) < 17 { + t.Fatalf("only %d routes in the table", len(rs)) + } + for i, rt := range rs { + method, pattern, _ := strings.Cut(rt.pattern, " ") + path := strings.NewReplacer("{id}", "1", "{address}", "a@example.com").Replace(pattern) + t.Run(rt.pattern, func(t *testing.T) { + // Each request comes from its own address, so the lockout never gets in the way. + addr := fmt.Sprintf("192.0.2.%d:4000", i+1) + call := func(auth bool) int { + req := httptest.NewRequest(method, path, strings.NewReader("{}")) + req.RemoteAddr = addr + if auth { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec.Code + } + if pattern == "/api/v1/health" { + if got := call(false); got != 200 { + t.Fatalf("health without a token: %d", got) + } + return + } + if got := call(false); got != 401 { + t.Errorf("no token: status %d, want 401", got) + } + if got := call(true); got == 401 { + t.Errorf("good token: status %d", got) + } + }) + } +} From b8fcbf9acf0d2260ed24422e31270053ae253697 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:50:40 +0100 Subject: [PATCH 2/9] fix(api): validate list name, domain and title on create createList accepted a title with CR/LF (every later post to the list then failed to rewrite and got 451) and names with '/', '<', '>' or tabs. - name: lowercased, 1 to 30 characters of a-z 0-9 . _ -, no leading or trailing dot, no ".." - domain: lowercased ASCII host name, labels of 1 to 63 characters - title: non-empty single line, at most 200 characters, no control characters - updateList uses the same title rule (it also refuses tabs and long titles now) - tests: refused names, domains and titles, the accepted limits, nothing created for a bad title --- internal/api/api.go | 72 ++++++++++++++++++++++++++++++++++---- internal/api/api_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 7 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index e9e7200..0c96bb3 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" "time" + "unicode" + "unicode/utf8" "codeberg.org/squidd/parsimony/internal/archive" "codeberg.org/squidd/parsimony/internal/store" @@ -184,6 +186,53 @@ func (s *server) listLists(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, out) } +const ( + maxNameLen = 30 // keeps list+confirm- and list+bounce-- within 64 characters + maxTitleLen = 200 + titleHint = "title must be a non-empty single line of at most 200 characters, without control characters" +) + +// validName reports whether name (already lowercased) works as the local part of a list address. +func validName(name string) bool { + if name == "" || len(name) > maxNameLen || strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".") || strings.Contains(name, "..") { + return false + } + for _, c := range name { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '.' && c != '_' && c != '-' { + return false + } + } + return true +} + +// validDomain reports whether d (already lowercased) is an ASCII host name: dot-separated labels of +// 1 to 63 letters, digits and hyphens, none starting or ending with a hyphen. +func validDomain(d string) bool { + if d == "" || len(d) > 253 { + return false + } + for _, label := range strings.Split(d, ".") { + if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for _, c := range label { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' { + return false + } + } + } + return true +} + +// validTitle reports whether t is a usable list title: not empty, at most maxTitleLen characters, +// no line breaks or other control characters (it goes into mail headers). +func validTitle(t string) bool { + if t == "" || utf8.RuneCountInString(t) > maxTitleLen { + return false + } + return !strings.ContainsFunc(t, unicode.IsControl) +} + func (s *server) createList(w http.ResponseWriter, r *http.Request) { var in struct { Name string `json:"name"` @@ -193,13 +242,22 @@ func (s *server) createList(w http.ResponseWriter, r *http.Request) { if !decodeBody(w, r, &in) { return } - in.Name, in.Domain = strings.TrimSpace(in.Name), strings.TrimSpace(in.Domain) - if in.Name == "" || in.Domain == "" || strings.ContainsAny(in.Name+in.Domain, "@+ \r\n") { - writeError(w, http.StatusBadRequest, "name and domain are required and must not contain '@', '+' or spaces") + typedName := strings.TrimSpace(in.Name) + in.Name, in.Domain = strings.ToLower(typedName), strings.ToLower(strings.TrimSpace(in.Domain)) + if !validName(in.Name) { + writeError(w, http.StatusBadRequest, "name must be 1 to 30 characters of a-z, 0-9, '.', '_' or '-', without a dot at either end or two dots in a row") return } - if in.Title == "" { - in.Title = in.Name + if !validDomain(in.Domain) { + writeError(w, http.StatusBadRequest, "domain must be a host name such as lists.example.org") + return + } + if in.Title = strings.TrimSpace(in.Title); in.Title == "" { + in.Title = typedName + } + if !validTitle(in.Title) { + writeError(w, http.StatusBadRequest, titleHint) + return } l, err := s.store.CreateList(in.Name, in.Domain, in.Title) if err != nil { @@ -248,8 +306,8 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { } if in.Title != nil { title := strings.TrimSpace(*in.Title) - if title == "" || strings.ContainsAny(title, "\r\n") { - writeError(w, http.StatusBadRequest, "title must be a non-empty single line") + if !validTitle(title) { + writeError(w, http.StatusBadRequest, titleHint) return } l.Title = title diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 984dda7..19ddfdf 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -152,6 +152,34 @@ func TestCreateListValidation(t *testing.T) { `{"name":"a+b","domain":"example.org"}`, `{"name":"dev","domain":""}`, `{"name":"dev","domain":"example.org","bogus":1}`, + // Names that break an address: separators, brackets, whitespace, control characters, dots at the ends. + `{"name":"a/b","domain":"example.org"}`, + `{"name":"ab","domain":"example.org"}`, + `{"name":"a\tb","domain":"example.org"}`, + `{"name":"a b","domain":"example.org"}`, + `{"name":"a\"b","domain":"example.org"}`, + `{"name":"caffè","domain":"example.org"}`, + `{"name":".dev","domain":"example.org"}`, + `{"name":"dev.","domain":"example.org"}`, + `{"name":"a..b","domain":"example.org"}`, + `{"name":"` + strings.Repeat("a", 31) + `","domain":"example.org"}`, + // Domains that are not host names. + `{"name":"dev","domain":"exa mple.org"}`, + `{"name":"dev","domain":"example..org"}`, + `{"name":"dev","domain":"-example.org"}`, + `{"name":"dev","domain":"example-.org"}`, + `{"name":"dev","domain":"example.org."}`, + `{"name":"dev","domain":"ex_ample.org"}`, + `{"name":"dev","domain":"exämple.org"}`, + `{"name":"dev","domain":"a/b.org"}`, + `{"name":"dev","domain":"` + strings.Repeat("a", 64) + `.org"}`, + // Titles that are not one clean line. + `{"name":"dev","domain":"example.org","title":"Dev\r\nBcc: x@y.com"}`, + `{"name":"dev","domain":"example.org","title":"Dev\nlist"}`, + `{"name":"dev","domain":"example.org","title":"Dev\tlist"}`, + `{"name":"dev","domain":"example.org","title":"Dev\u0000list"}`, + `{"name":"dev","domain":"example.org","title":"` + strings.Repeat("x", 201) + `"}`, } { rec := do(t, h, "POST", "/api/v1/lists", body, true) if rec.Code != 400 { @@ -732,3 +760,49 @@ func TestEveryRouteNeedsTheToken(t *testing.T) { }) } } + +func TestCreateListAcceptsTheLimits(t *testing.T) { + h := newServer(t) + for _, body := range []string{ + `{"name":"` + strings.Repeat("a", 30) + `","domain":"example.org","title":"` + strings.Repeat("è", 200) + `"}`, + `{"name":"dev-team_1.eu","domain":"lists.sub-domain.example.org"}`, + `{"name":"Dev2","domain":"localhost","title":"Développeurs ☕"}`, + } { + if rec := do(t, h, "POST", "/api/v1/lists", body, true); rec.Code != 201 { + t.Errorf("%.80q: status %d, body %s", body, rec.Code, rec.Body.String()) + } + } +} + +// A title with a line break would make every later post to the list fail to rewrite, so the list +// must not come into existence at all. +func TestCreateListWithABadTitleLeavesNothingBehind(t *testing.T) { + h := newServer(t) + expect(t, do(t, h, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev\r\nX: y"}`, true), 400) + rec := do(t, h, "GET", "/api/v1/lists", "", true) + if strings.TrimSpace(rec.Body.String()) != "[]" { + t.Fatalf("the refused list was created: %s", rec.Body.String()) + } + // The name is free for a good request. + expect(t, do(t, h, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, true), 201) +} + +func TestCreateListLowercasesNameAndDomain(t *testing.T) { + h := newServer(t) + rec := do(t, h, "POST", "/api/v1/lists", `{"name":" Dev ","domain":"Example.ORG"}`, true) + expect(t, rec, 201) + var l listJSON + decode(t, rec, &l) + if l.Address != "dev@example.org" || l.Title != "Dev" { + t.Fatalf("%+v", l) + } +} + +func TestUpdateListTitleUsesTheSameRules(t *testing.T) { + h := newServer(t) + createList(t, h) + for _, title := range []string{`"a\tb"`, `"` + strings.Repeat("x", 201) + `"`, `" "`, `"a\r\nb"`} { + expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"title":`+title+`}`, true), 400) + } + expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"title":"`+strings.Repeat("x", 200)+`"}`, true), 200) +} From 087bc92d413911e856281857bf9268172e1f6829 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:51:05 +0100 Subject: [PATCH 3/9] fix(router): RFC 2047 encode a non-ASCII List-Id phrase - headers: the title in List-Id goes through mime.QEncoding, the part stays ASCII - test: UTF-8 and 200-character titles read back through net/mail and the word decoder --- internal/router/headers.go | 4 ++- internal/router/router_test.go | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/internal/router/headers.go b/internal/router/headers.go index 8847dea..c9f5a3c 100644 --- a/internal/router/headers.go +++ b/internal/router/headers.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "io" + "mime" "net/mail" "github.com/emersion/go-message/textproto" @@ -27,7 +28,8 @@ func rewrite(l store.List, data []byte) ([]byte, error) { } id := l.Name + "." + l.Domain - h.Set("List-Id", fmt.Sprintf("%s <%s>", l.Title, id)) + // The phrase is RFC 2047 encoded when it is not ASCII; the stays plain ASCII (list names and domains are). + h.Set("List-Id", fmt.Sprintf("%s <%s>", mime.QEncoding.Encode("utf-8", l.Title), id)) h.Set("List-Post", "") // List-Unsubscribe is per recipient: Router.Send adds it. h.Set("Precedence", "list") diff --git a/internal/router/router_test.go b/internal/router/router_test.go index 1cb7856..5e12516 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -3,6 +3,7 @@ package router import ( "bytes" "errors" + "mime" "net/mail" "strings" "testing" @@ -107,3 +108,51 @@ func TestRewriteKeepsExistingReplyTo(t *testing.T) { t.Errorf("Reply-To = %q", got) } } + +func TestListIdWithANonASCIITitleIsEncoded(t *testing.T) { + l := store.List{Name: "dev", Domain: "example.org", Title: "Développeurs ☕ \"café\""} + out, err := rewrite(l, []byte(sample)) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(out), "\r\n") { + if strings.HasPrefix(line, "List-Id:") { + for _, b := range []byte(line) { + if b >= 0x80 { + t.Fatalf("raw non-ASCII in %q", line) + } + } + } + } + m, err := mail.ReadMessage(bytes.NewReader(out)) + if err != nil { + t.Fatal(err) + } + raw := m.Header.Get("List-Id") + if !strings.HasSuffix(raw, " ") { + t.Fatalf("the id must stay a plain : %q", raw) + } + phrase, err := new(mime.WordDecoder).DecodeHeader(strings.TrimSuffix(raw, " ")) + if err != nil || phrase != l.Title { + t.Fatalf("phrase = %q (%v), want %q", phrase, err, l.Title) + } + // A long title is folded into several encoded words and still reads back. + long := store.List{Name: "dev", Domain: "example.org", Title: strings.Repeat("è", 200)} + out, err = rewrite(long, []byte(sample)) + if err != nil { + t.Fatal(err) + } + m, err = mail.ReadMessage(bytes.NewReader(out)) + if err != nil { + t.Fatal(err) + } + raw = m.Header.Get("List-Id") + if phrase, err := new(mime.WordDecoder).DecodeHeader(strings.TrimSuffix(raw, " ")); err != nil || phrase != long.Title { + t.Fatalf("long phrase = %q (%v)", phrase, err) + } + // An ASCII title is left as it is. + out, _ = rewrite(store.List{Name: "dev", Domain: "example.org", Title: "Dev"}, []byte(sample)) + if !strings.Contains(string(out), "List-Id: Dev \r\n") { + t.Fatalf("ascii title changed: %q", out) + } +} From f340078aed80c9c654d358210b7b11b70b17e014 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:51:38 +0100 Subject: [PATCH 4/9] fix(config): check base_url at load - config: empty is fine; otherwise https with a host, http only for localhost/127.0.0.1/::1, no userinfo, query, fragment or whitespace; the error names base_url - test: accepted and refused URLs - docs: README states the rule --- README.md | 4 +++- internal/config/config.go | 26 +++++++++++++++++++++++ internal/config/config_test.go | 38 ++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 61453bc..bb46e26 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,9 @@ Three listeners, each configurable: | `web` | `127.0.0.1:8081` | public archive and unsubscribe pages (`""` disables) | Put the `web` listener behind a reverse proxy with TLS and set `base_url` to its public URL: -the unsubscribe links in every message use it. Keep the `api` listener private. +the unsubscribe links in every message use it. It must be an `https://` URL (RFC 8058 one-click unsubscribe needs +it; `http://` is accepted only for `localhost`, `127.0.0.1` and `::1`) with a host and no credentials, query or fragment; +the daemon refuses to start otherwise. Leave it empty to turn the one-click links off. Keep the `api` listener private. ``` curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists diff --git a/internal/config/config.go b/internal/config/config.go index 76148d1..569048d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,8 +7,10 @@ import ( "errors" "fmt" "net" + "net/url" "os" "strings" + "unicode" "gopkg.in/yaml.v3" ) @@ -145,7 +147,31 @@ func (c Config) validateRelay() error { return nil } +// validateBaseURL checks base_url, the public URL the one-click unsubscribe links are built from. +// RFC 8058 needs https; http is accepted for a loopback host (development and tests). Empty is fine: +// there is no one-click link then. +func (c Config) validateBaseURL() error { + if c.BaseURL == "" { + return nil + } + u, err := url.Parse(c.BaseURL) + switch { + case err != nil || strings.ContainsFunc(c.BaseURL, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }): + return fmt.Errorf("base_url %q is not a valid URL", c.BaseURL) + case u.Hostname() == "": + return fmt.Errorf("base_url %q needs a scheme and a host, such as https://lists.example.org", c.BaseURL) + case u.Scheme != "https" && !(u.Scheme == "http" && isLoopbackHost(u.Hostname())): + return fmt.Errorf("base_url %q must use https (one-click unsubscribe, RFC 8058); http is only accepted for localhost", c.BaseURL) + case u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(c.BaseURL, "#"): + return fmt.Errorf("base_url %q must not contain credentials, a query or a fragment", c.BaseURL) + } + return nil +} + func (c Config) validate() error { + if err := c.validateBaseURL(); err != nil { + return err + } if err := c.validateRelay(); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e08035..4381983 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -246,3 +246,41 @@ func testCAPEM(t *testing.T) []byte { } return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) } + +func TestBaseURLIsChecked(t *testing.T) { + for _, ok := range []string{ + "", // one-click unsubscribe is simply off + "https://lists.example.org", + "https://lists.example.org/", + "https://lists.example.org:8443/sub/path", + "http://localhost:8081", + "http://127.0.0.1:8081", + "http://[::1]:8081", + } { + if _, err := Load(write(t, "base_url: \""+ok+"\"\n")); err != nil { + t.Errorf("%q must be accepted: %v", ok, err) + } + } + for _, bad := range []string{ + "http://lists.example.org", // RFC 8058 needs https + "http://10.0.0.5:8081", // only loopback may use http + "ftp://lists.example.org", // not http(s) at all + "lists.example.org", // no scheme + "//lists.example.org", // no scheme + "https://", // no host + "https:///path", // no host + "https://lists.example.org?x=1", // query + "https://lists.example.org/#top", // fragment + "https://user:pw@lists.example.org", // userinfo + "https://user@lists.example.org", + "https://lists.example.org/a b", + "https://exa mple.org", + } { + _, err := Load(write(t, "base_url: \""+bad+"\"\n")) + if err == nil { + t.Errorf("%q must be rejected", bad) + } else if !strings.Contains(err.Error(), "base_url") { + t.Errorf("%q: the error must name base_url: %v", bad, err) + } + } +} From 1e8d60b72c1d129778db28c0c7e4b8793a456ba7 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:52:42 +0100 Subject: [PATCH 5/9] fix(store): run the v1 members migration in one transaction - store: migrateMembersV1 renames, creates, copies and drops inside a transaction - test: a copy that fails on an orphan row leaves the old members table with its rows and no members_v1 --- internal/store/m2_test.go | 44 +++++++++++++++++++++++++++++++++++++++ internal/store/store.go | 32 ++++++++++++++++++++-------- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index d1f03d0..5af346c 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -467,3 +467,47 @@ func TestThreadListingsDoNotLoadBodies(t *testing.T) { t.Fatal("a single message still has its body") } } + +// The members migration renames, creates, copies and drops. If any step fails, nothing may be left +// half done: the old table keeps its rows, and no members_v1 stays behind. +func TestFailedMembersMigrationLeavesTheOldTablesIntact(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + // The second member belongs to a list that does not exist: the copy fails on the foreign key. + _, err = db.Exec(` +CREATE TABLE lists (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, domain TEXT NOT NULL, title TEXT NOT NULL DEFAULT '', UNIQUE(name, domain)); +CREATE TABLE members (list_id INTEGER NOT NULL, address TEXT NOT NULL, display_name TEXT NOT NULL DEFAULT '', PRIMARY KEY(list_id, address)); +INSERT INTO lists(name, domain, title) VALUES('dev','example.org','Dev'); +INSERT INTO members(list_id, address, display_name) VALUES(1,'a@x.com','A'),(99,'ghost@x.com','G');`) + if err != nil { + t.Fatal(err) + } + db.Close() + + if s, err := Open(path); err == nil { + s.Close() + t.Fatal("the migration must fail on the orphan row") + } + + db, err = sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var n int + db.QueryRow(`SELECT COUNT(*) FROM members`).Scan(&n) + if n != 2 { + t.Fatalf("the old members table lost rows: %d", n) + } + if has, _ := hasColumn(db, "members", "id"); has { + t.Fatal("members must still be the old table") + } + var tables int + db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name='members_v1'`).Scan(&tables) + if tables != 0 { + t.Fatal("members_v1 was left behind") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 5abaedf..38581ec 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -246,15 +246,7 @@ func migrate(db *sql.DB) error { if hasID, err := hasColumn(db, "members", "id"); err != nil { return err } else if !hasID { - if _, err := db.Exec(`ALTER TABLE members RENAME TO members_v1`); err != nil { - return err - } - if _, err := db.Exec(schema); err != nil { - return err - } - if _, err := db.Exec(`INSERT INTO members(list_id, address, display_name) - SELECT list_id, address, display_name FROM members_v1 ORDER BY list_id, address; - DROP TABLE members_v1;`); err != nil { + if err := migrateMembersV1(db); err != nil { return err } } @@ -305,6 +297,28 @@ func migrate(db *sql.DB) error { return err } +// migrateMembersV1 replaces the milestone 1 members table (no id column) in one transaction, so a +// failure or a crash in the middle leaves the old table exactly as it was. +func migrateMembersV1(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`ALTER TABLE members RENAME TO members_v1`); err != nil { + return err + } + if _, err := tx.Exec(schema); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO members(list_id, address, display_name) + SELECT list_id, address, display_name FROM members_v1 ORDER BY list_id, address; + DROP TABLE members_v1;`); err != nil { + return err + } + return tx.Commit() +} + // addHeldSummary adds the summary columns of the held queue and fills them for the messages // already there, one message at a time so that a big queue is not loaded into memory. func addHeldSummary(db *sql.DB) error { From cfcfd42110c2e59a2405b2521797331d676b53d3 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:54:05 +0100 Subject: [PATCH 6/9] fix(router,store): consume a confirmation token only when its action happened - store: ConfirmPending claims the row (DELETE ... RETURNING), applies join or leave and commits in one transaction; a refusal from check, a database error, a token of another list or a rollback leaves the token usable; concurrent confirmations of one token apply exactly once - store: CreatePending purges pending rows older than the 72 h TTL - store: IsActiveMember - router: confirm uses it, the list policy check runs before anything is consumed - router: +join from a member disabled by bounces sends the confirmation; confirming re-enables it - tests: purge, refusal, forced database error, other list, expiry, 8 concurrent confirms, whole rejoin flow - cli test: the create error text changed with the new name rule --- internal/cli/cli_test.go | 2 +- internal/router/commands.go | 30 ++--- internal/router/m2_test.go | 75 +++++++++++++ internal/store/m2_test.go | 211 +++++++++++++++++++++++++++++++++++- internal/store/store.go | 71 ++++++++++-- 5 files changed, 357 insertions(+), 32 deletions(-) diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index ff494f8..d205820 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -153,7 +153,7 @@ func TestListLifecycle(t *testing.T) { e.fails(t, 1, "already exists", "list", "add", "dev@example.org") e.fails(t, 2, "name@domain", "list", "add", "nodomain") e.fails(t, 2, "name@domain", "list", "add", "@example.org") - e.fails(t, 1, "name and domain", "list", "add", "a+b@example.org") + e.fails(t, 1, "name must be", "list", "add", "a+b@example.org") for _, ref := range []string{"dev@example.org", "DEV@Example.org", "1"} { out = e.ok(t, "list", "show", ref) diff --git a/internal/router/commands.go b/internal/router/commands.go index 499c303..d9628e7 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -85,8 +85,9 @@ func (r *Router) join(l store.List, sender string) error { if l.JoinPolicy != store.JoinOpen { return ErrJoinClosed } - if ok, err := r.store.IsMember(l.ID, sender); err != nil || ok { - return err // members get no mail: it would leak membership + // A member disabled by bounces is treated like a non-member: confirming re-enables it. + if ok, err := r.store.IsActiveMember(l.ID, sender); err != nil || ok { + return err // active members get no mail: it would leak membership } return r.askConfirmation(l, sender, store.ActionJoin) } @@ -126,25 +127,18 @@ func (r *Router) askConfirmation(l store.List, addr, action string) error { } func (r *Router) confirm(l store.List, token string) error { - p, err := r.store.ConfirmPending(token) - if errors.Is(err, store.ErrNotFound) || (err == nil && p.ListID != l.ID) { - return ErrBadToken - } - if err != nil { - return err - } - switch p.Action { - case store.ActionJoin: - if l.JoinPolicy != store.JoinOpen { + // The token is used up only if the action happens: a closed list or a database error leaves it + // valid, and two replies to the same mail cannot both apply. + _, err := r.store.ConfirmPending(token, l.ID, func(p store.Pending) error { + if p.Action == store.ActionJoin && l.JoinPolicy != store.JoinOpen { return ErrJoinClosed } - return r.store.AddMember(l.ID, p.Address, "") - case store.ActionLeave: - if err := r.store.RemoveMember(l.ID, p.Address); err != nil && !errors.Is(err, store.ErrNotFound) { - return err - } + return nil + }) + if errors.Is(err, store.ErrNotFound) { + return ErrBadToken } - return nil + return err } func (r *Router) bounce(l store.List, tag string, data []byte) error { diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index 1a1d1c6..669a6e6 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -236,6 +236,81 @@ func TestConfirmAfterListClosed(t *testing.T) { if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok { t.Fatal("must not join a closed list") } + // The refusal did not use the token up: once the list is open again the same mail works. + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinOpen, store.ArchivePrivate) + if _, err := e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil); err != nil { + t.Fatalf("the token must survive a refusal: %v", err) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok { + t.Fatal("carol must join once the list is open") + } +} + +func TestConfirmWithTokenOfAnotherListKeepsItUsable(t *testing.T) { + e := newEnv(t) + e.s.CreateList("ops", "example.org", "Ops") + e.command("carol@x.com", "dev+join@example.org", nil) + tok := token(t, lastMail(t, e.out)) + if _, err := e.command("carol@x.com", "ops+confirm-"+tok+"@example.org", nil); !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } + if _, err := e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil); err != nil { + t.Fatalf("the right list still accepts it: %v", err) + } +} + +// A member disabled by bounces must be able to rejoin: +join sends the confirmation, confirming +// re-enables the subscription and clears the bounce state. +func TestDisabledMemberCanRejoin(t *testing.T) { + e := newEnv(t) + for i := 0; i < store.BounceThreshold; i++ { + e.s.RecordHardBounce(e.alice, store.BounceThreshold) + } + if m, _ := e.s.MemberByID(e.alice); !m.Disabled { + t.Fatal("setup: alice must be disabled") + } + n := len(e.out.Sent) + if _, err := e.command("alice@example.com", "dev+join@example.org", nil); err != nil { + t.Fatal(err) + } + if len(e.out.Sent) != n+1 { + t.Fatalf("a disabled member must get the confirmation, sent %d new mails", len(e.out.Sent)-n) + } + m := lastMail(t, e.out) + if len(m.To) != 1 || m.To[0] != "alice@example.com" { + t.Fatalf("to = %v", m.To) + } + if got, _ := e.s.MemberByID(e.alice); !got.Disabled { + t.Fatal("not re-enabled before confirming") + } + if _, err := e.command("alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil); err != nil { + t.Fatal(err) + } + got, err := e.s.MemberByID(e.alice) + if err != nil || got.Disabled || got.BounceCount != 0 || got.ID != e.alice { + t.Fatalf("after confirming: %+v %v", got, err) + } + // She receives list mail again. + res, err := e.r.Route("bob@example.com", "dev@example.org", postFrom("Bob ")) + if err != nil { + t.Fatal(err) + } + seen := false + for _, r := range res.Recipients { + seen = seen || r.Address == "alice@example.com" + } + if !seen { + t.Fatalf("alice must receive posts again: %+v", res.Recipients) + } +} + +// An active member still gets nothing: no mail that would reveal membership. +func TestJoinStaysSilentForActiveMembers(t *testing.T) { + e := newEnv(t) + e.command("alice@example.com", "dev+join@example.org", nil) + if len(e.out.Sent) != 0 { + t.Fatal("active members get no mail") + } } func TestLeaveFlow(t *testing.T) { diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 5af346c..0c71628 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -6,6 +6,8 @@ import ( "fmt" "path/filepath" "strings" + "sync" + "sync/atomic" "testing" "time" ) @@ -82,20 +84,20 @@ func TestPendingTokens(t *testing.T) { if _, created, _ := s.CreatePending(l.ID, "a@x.com", "leave"); !created { t.Fatal("leave is a separate request") } - p, err := s.ConfirmPending(tok) + p, err := s.ConfirmPending(tok, l.ID, nil) if err != nil || p.Address != "a@x.com" || p.Action != "join" || p.ListID != l.ID { t.Fatalf("confirm: %+v %v", p, err) } - if _, err := s.ConfirmPending(tok); !errors.Is(err, ErrNotFound) { + if _, err := s.ConfirmPending(tok, l.ID, nil); !errors.Is(err, ErrNotFound) { t.Fatal("token is single use") } - if _, err := s.ConfirmPending("nope"); !errors.Is(err, ErrNotFound) { + if _, err := s.ConfirmPending("nope", l.ID, nil); !errors.Is(err, ErrNotFound) { t.Fatal("unknown token") } // Expiry. tok2, _, _ := s.CreatePending(l.ID, "b@x.com", "join") now = now.Add(73 * time.Hour) - if _, err := s.ConfirmPending(tok2); !errors.Is(err, ErrNotFound) { + if _, err := s.ConfirmPending(tok2, l.ID, nil); !errors.Is(err, ErrNotFound) { t.Fatal("token must expire after 72h") } } @@ -511,3 +513,204 @@ INSERT INTO members(list_id, address, display_name) VALUES(1,'a@x.com','A'),(99, t.Fatal("members_v1 was left behind") } } + +func pendingRows(t *testing.T, s *Store) int { + t.Helper() + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM pending`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestCreatePendingPurgesExpiredRows(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.CreatePending(l.ID, "old@x.com", "join") + now = now.Add(70 * time.Hour) + s.CreatePending(l.ID, "recent@x.com", "join") + if n := pendingRows(t, s); n != 2 { + t.Fatalf("rows within the TTL must stay: %d", n) + } + // 72h after the first request (and a little more): it goes, the other stays. + now = now.Add(3 * time.Hour) + s.CreatePending(l.ID, "third@x.com", "join") + if n := pendingRows(t, s); n != 2 { + t.Fatalf("the expired row must be purged: %d rows", n) + } + var old int + s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address='old@x.com'`).Scan(&old) + if old != 0 { + t.Fatal("the expired request is still there") + } + // Exactly at the TTL a token is still valid, so it is not purged either. + now = now.Add(pendingTTL) + s.CreatePending(l.ID, "fourth@x.com", "join") + var recent int + s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address='third@x.com'`).Scan(&recent) + if recent != 1 { + t.Fatal("a row exactly at the TTL must not be purged yet") + } +} + +func TestConfirmPendingAppliesTheActionAndConsumesTheToken(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + tok, _, _ := s.CreatePending(l.ID, "carol@x.com", ActionJoin) + p, err := s.ConfirmPending(tok, l.ID, nil) + if err != nil || p.Address != "carol@x.com" || p.Action != ActionJoin { + t.Fatalf("%+v %v", p, err) + } + if ok, _ := s.IsMember(l.ID, "carol@x.com"); !ok { + t.Fatal("join must subscribe") + } + if pendingRows(t, s) != 0 { + t.Fatal("the token must be consumed") + } + + tok, _, _ = s.CreatePending(l.ID, "carol@x.com", ActionLeave) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatal(err) + } + if ok, _ := s.IsMember(l.ID, "carol@x.com"); ok { + t.Fatal("leave must unsubscribe") + } + // Leaving when already gone still succeeds and still uses the token. + tok, _, _ = s.CreatePending(l.ID, "ghost@x.com", ActionLeave) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatalf("leave of a non-member: %v", err) + } + if pendingRows(t, s) != 0 { + t.Fatal("the token must be consumed") + } +} + +func TestConfirmPendingJoinReenablesADisabledMember(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "bob@x.com", "Bob") + ms, _ := s.Members(l.ID) + for i := 0; i < BounceThreshold; i++ { + s.RecordHardBounce(ms[0].ID, BounceThreshold) + } + tok, _, _ := s.CreatePending(l.ID, "bob@x.com", ActionJoin) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatal(err) + } + m, _ := s.MemberByID(ms[0].ID) + if m.Disabled || m.BounceCount != 0 { + t.Fatalf("%+v", m) + } +} + +// A refusal or a failure leaves the token usable: the confirmation is not lost. +func TestConfirmPendingKeepsTheTokenWhenTheActionDoesNotHappen(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + tok, _, _ := s.CreatePending(l.ID, "carol@x.com", ActionJoin) + + refused := errors.New("closed") + if _, err := s.ConfirmPending(tok, l.ID, func(Pending) error { return refused }); !errors.Is(err, refused) { + t.Fatalf("want the check's error, got %v", err) + } + if pendingRows(t, s) != 1 { + t.Fatal("a refused confirmation must keep its token") + } + + // A database error while subscribing: force one with a trigger. + if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON members BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil { + t.Fatal(err) + } + if _, err := s.ConfirmPending(tok, l.ID, nil); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("want the database error, got %v", err) + } + if pendingRows(t, s) != 1 { + t.Fatal("a failed AddMember must keep the token") + } + s.db.Exec(`DROP TRIGGER boom`) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatalf("the token must still work: %v", err) + } + if ok, _ := s.IsMember(l.ID, "carol@x.com"); !ok { + t.Fatal("carol must be subscribed") + } +} + +func TestConfirmPendingForAnotherListIsNotFoundAndKeepsTheToken(t *testing.T) { + s := newTestStore(t) + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + tok, _, _ := s.CreatePending(dev.ID, "carol@x.com", ActionJoin) + if _, err := s.ConfirmPending(tok, ops.ID, nil); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if ok, _ := s.IsMember(ops.ID, "carol@x.com"); ok { + t.Fatal("must not join the wrong list") + } + if _, err := s.ConfirmPending(tok, dev.ID, nil); err != nil { + t.Fatalf("the right list can still use it: %v", err) + } +} + +func TestConfirmPendingExpiredIsNotFound(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + tok, _, _ := s.CreatePending(l.ID, "carol@x.com", ActionJoin) + now = now.Add(pendingTTL) // exactly at the limit: still valid + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatalf("at the TTL: %v", err) + } + tok, _, _ = s.CreatePending(l.ID, "dave@x.com", ActionJoin) + now = now.Add(pendingTTL + time.Second) + checked := false + if _, err := s.ConfirmPending(tok, l.ID, func(Pending) error { checked = true; return nil }); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if checked { + t.Fatal("an expired token must not reach the check") + } + if ok, _ := s.IsMember(l.ID, "dave@x.com"); ok { + t.Fatal("an expired token must not subscribe") + } +} + +// Two confirmations of one token at the same time: exactly one applies. +func TestConfirmPendingIsSingleUseUnderConcurrency(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "p.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + for round := 0; round < 20; round++ { + addr := fmt.Sprintf("u%d@x.com", round) + tok, _, _ := s.CreatePending(l.ID, addr, ActionJoin) + var applied, notFound, other atomic.Int32 + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := s.ConfirmPending(tok, l.ID, func(Pending) error { applied.Add(1); return nil }) + switch { + case err == nil: + case errors.Is(err, ErrNotFound): + notFound.Add(1) + default: + other.Add(1) + } + }() + } + close(start) + wg.Wait() + if applied.Load() != 1 || notFound.Load() != 7 || other.Load() != 0 { + t.Fatalf("round %d: applied %d, not found %d, other errors %d", round, applied.Load(), notFound.Load(), other.Load()) + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 38581ec..832aa38 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -503,7 +503,11 @@ func scanMember(row interface{ Scan(...any) error }) (Member, error) { // AddMember subscribes an address to a list. Re-adding keeps the member id and // clears the bounce state. func (s *Store) AddMember(listID int64, addr, displayName string) error { - _, err := s.db.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?) + return addMember(s.db, listID, addr, displayName) +} + +func addMember(q querier, listID int64, addr, displayName string) error { + _, err := q.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?) ON CONFLICT(list_id, address) DO UPDATE SET display_name=excluded.display_name, disabled=0, bounce_count=0`, listID, norm(addr), displayName) return err @@ -516,6 +520,13 @@ func (s *Store) IsMember(listID int64, addr string) (bool, error) { return n > 0, err } +// IsActiveMember reports whether addr is subscribed and not disabled by bounces. +func (s *Store) IsActiveMember(listID int64, addr string) (bool, error) { + var n int + err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=? AND disabled=0`, listID, norm(addr)).Scan(&n) + return n > 0, err +} + func (s *Store) queryMembers(where string, args ...any) ([]Member, error) { rows, err := s.db.Query(`SELECT `+memberCols+` FROM members WHERE `+where+` ORDER BY address`, args...) if err != nil { @@ -550,7 +561,11 @@ func (s *Store) MemberByID(id int64) (Member, error) { // RemoveMember unsubscribes an address from a list. func (s *Store) RemoveMember(listID int64, addr string) error { - return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr))) + return removeMember(s.db, listID, addr) +} + +func removeMember(q querier, listID int64, addr string) error { + return affected(q.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr))) } // DeleteMember unsubscribes a member by id. @@ -666,6 +681,11 @@ func randomHex(n int) (string, error) { func (s *Store) CreatePending(listID int64, addr, action string) (token string, created bool, err error) { addr = norm(addr) now := s.Now() + // Expired requests are dead weight (a token older than the TTL is refused): drop them here, so + // the table stays small without a background job. + if _, err = s.db.Exec(`DELETE FROM pending WHERE created_at?`, listID, addr, action, now.Add(-pendingRateLimit).Unix()).Scan(&n) @@ -691,23 +711,56 @@ func (s *Store) CreatePending(listID int64, addr, action string) (token string, return token, true, nil } -// ConfirmPending consumes a token. Unknown, used and expired tokens give ErrNotFound. -func (s *Store) ConfirmPending(token string) (Pending, error) { +// ConfirmPending applies the request behind a token and consumes the token, in one transaction: a +// token is only used up when its action happened, and two confirmations of the same token cannot +// both apply (the first one to claim the row wins, the other gets ErrNotFound). +// +// Unknown, used, expired tokens and tokens of another list give ErrNotFound (the last two are left +// alone). check, when not nil, runs before anything changes; its error is returned and the token +// stays valid. A "join" subscribes the address (re-enabling a disabled member), a "leave" +// unsubscribes it (already gone is fine). +func (s *Store) ConfirmPending(token string, listID int64, check func(Pending) error) (Pending, error) { + tx, err := s.db.Begin() + if err != nil { + return Pending{}, err + } + defer tx.Rollback() var p Pending var ts int64 - err := s.db.QueryRow(`SELECT token, list_id, address, action, created_at FROM pending WHERE token=?`, token). - Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts) + // DELETE ... RETURNING takes the write lock first, so concurrent confirmations queue up here. + err = tx.QueryRow(`DELETE FROM pending WHERE token=? AND list_id=? RETURNING token, list_id, address, action, created_at`, + token, listID).Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts) if errors.Is(err, sql.ErrNoRows) { return Pending{}, ErrNotFound } if err != nil { return Pending{}, err } - if _, err := s.db.Exec(`DELETE FROM pending WHERE token=?`, token); err != nil { + if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL { + // Expired: the row is gone for good. + if err := tx.Commit(); err != nil { + return Pending{}, err + } + return Pending{}, ErrNotFound + } + if check != nil { + if err := check(p); err != nil { + return Pending{}, err + } + } + switch p.Action { + case ActionJoin: + err = addMember(tx, p.ListID, p.Address, "") + case ActionLeave: + if err = removeMember(tx, p.ListID, p.Address); errors.Is(err, ErrNotFound) { + err = nil + } + } + if err != nil { return Pending{}, err } - if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL { - return Pending{}, ErrNotFound + if err := tx.Commit(); err != nil { + return Pending{}, err } return p, nil } From 555578bac55d3cf98f3e88deb3db2d0678032e5d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:54:28 +0100 Subject: [PATCH 7/9] fix: shorten the confirmation token to 20 hex characters name(<=30) + "+confirm-" + 20 = 59 characters, inside the 64 of a local part (a 30-character name with the old 32-digit token made 71). - store: CreatePending makes an 80-bit token - tests: a 30-character list name keeps the confirm and the VERP bounce address (12-digit id) within 64, and the shorter token still confirms --- internal/router/m2_test.go | 61 ++++++++++++++++++++++++++++++++++++++ internal/store/store.go | 4 ++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index 669a6e6..5d934cb 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -510,3 +510,64 @@ func TestIncomingListHeadersAreStripped(t *testing.T) { t.Errorf("Precedence = %v", h["Precedence"]) } } + +// A local part is at most 64 characters (RFC 5321): with the longest list name the API accepts, the +// confirm and the VERP bounce addresses must still fit. +func TestCommandAddressesFitTheLocalPartLimitForALongListName(t *testing.T) { + e := newEnv(t) + name := strings.Repeat("a", 30) + l, err := e.s.CreateList(name, "example.org", "Long") + if err != nil { + t.Fatal(err) + } + e.s.AddMember(l.ID, "alice@example.com", "Alice") + + if _, err := e.command("carol@x.com", name+"+join@example.org", nil); err != nil { + t.Fatal(err) + } + m := lastMail(t, e.out) + const marker = "Reply-To: " + i := strings.Index(string(m.Data), marker) + if i < 0 { + t.Fatalf("no Reply-To in %q", m.Data) + } + reply := strings.SplitN(string(m.Data)[i+len(marker):], "\r\n", 2)[0] + local, _, _ := strings.Cut(reply, "@") + if len(local) > 64 { + t.Fatalf("confirm local part is %d characters: %s", len(local), local) + } + tok := strings.TrimPrefix(local, name+"+confirm-") + if len(tok) != 20 { + t.Fatalf("token %q has %d characters, want 20 hex digits", tok, len(tok)) + } + // The shortened token still confirms. + if _, err := e.command("carol@x.com", reply, nil); err != nil { + t.Fatalf("confirm: %v", err) + } + if ok, _ := e.s.IsMember(l.ID, "carol@x.com"); !ok { + t.Fatal("carol must be a member") + } + + // VERP: the member id can have up to 12 digits. + for _, id := range []int64{1, 123456789012} { + addr := e.r.bounceAddress(l, id) + local, _, _ := strings.Cut(addr, "@") + if len(local) > 64 { + t.Fatalf("bounce local part for id %d is %d characters: %s", id, len(local), local) + } + } +} + +func TestConfirmTokensAreUnpredictable(t *testing.T) { + s, _ := store.Open(":memory:") + defer s.Close() + l, _ := s.CreateList("dev", "example.org", "Dev") + seen := map[string]bool{} + for i := 0; i < 50; i++ { + tok, created, err := s.CreatePending(l.ID, fmt.Sprintf("u%d@x.com", i), store.ActionJoin) + if err != nil || !created || len(tok) != 20 || seen[tok] { + t.Fatalf("token %q created=%v err=%v", tok, created, err) + } + seen[tok] = true + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 832aa38..873ab92 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -699,7 +699,9 @@ func (s *Store) CreatePending(listID int64, addr, action string) (token string, if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil || n >= maxPendingPerHour { return "", false, err } - token, err = randomHex(16) + // 80 bits: unguessable within the 72 h life of a token, and short enough that + // name(<=30)+"+confirm-"+token fits the 64 characters of a local part. + token, err = randomHex(10) if err != nil { return "", false, err } From 53ca30ad61aab7654f525b7be9f3c83581586c9e Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:54:52 +0100 Subject: [PATCH 8/9] docs: list name, domain and title rules, confirmation behaviour - openapi: name, domain and title constraints of POST /lists, title of PATCH - README: the rules, the RFC 2047 List-Id, single-use and re-usable confirmation, rejoin after bounces --- README.md | 8 +++++++- docs/openapi.yaml | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bb46e26..9bcc97e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,9 @@ A modern, single-binary mailing list manager: an evolution of Mailman. and a background worker sends it with retries (1m, 5m, 15m, 1h, 3h, then every 6h, for up to 3 days). Relay outages and restarts lose nothing. - Self-service subscribe and unsubscribe by email with double opt-in: - send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. + send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. A reply works once, a request + expires after 72 hours, and a confirmation that could not be applied (closed list, database error) can be replied to again. + A member disabled by bounces can rejoin the same way. - Moderated lists: posts wait in a queue that the API can approve or discard. - VERP bounces: every recipient gets a signed return address; 3 hard bounces disable a member. - Archive with full-text search (SQLite FTS5) and threading. Each list has an @@ -49,6 +51,10 @@ the daemon refuses to start otherwise. Leave it empty to turn the one-click link curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists ``` +A list `name` is 1 to 30 characters of `a-z`, `0-9`, `.`, `_` and `-` (upper case is lowered, no dot at either end), the `domain` +a host name, and the `title` one line of at most 200 characters without control characters; anything else is a 400. +The title is encoded (RFC 2047) in the `List-Id` header when it is not ASCII. + ## Command line The same binary is an administration client. It talks to the REST API, so it works against a local or remote daemon. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index d5b334b..2dc0d6e 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -99,9 +99,19 @@ paths: type: object required: [name, domain] properties: - name: {type: string} - domain: {type: string} - title: {type: string} + name: + type: string + minLength: 1 + maxLength: 30 + pattern: '^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$' + description: Local part of the list address, stored lowercase. Letters, digits, '.', '_' and '-' only, no dot at either end or two in a row. + domain: + type: string + description: ASCII host name (labels of 1 to 63 letters, digits and hyphens). + title: + type: string + maxLength: 200 + description: A single line without control characters. Defaults to the name. responses: '201': description: Created @@ -131,7 +141,7 @@ paths: schema: type: object properties: - title: {type: string} + title: {type: string, minLength: 1, maxLength: 200, description: A single line without control characters.} post_policy: {type: string, enum: [members, moderated]} join_policy: {type: string, enum: [open, closed]} archive_policy: {type: string, enum: [public, private, off]} From f7ce18c3d4e51408637f0e6f1bca174ff1e6bcb2 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:57:49 +0100 Subject: [PATCH 9/9] test: close the gaps found by mutation checks - api: empty names, '{' in a name, 253/254-character domains, title trimming and the name fallback - config: an empty query or fragment in base_url - router: leaving works on a list closed to new members - store: an expired token is removed when presented --- internal/api/api_test.go | 34 ++++++++++++++++++++++++++++++++++ internal/config/config_test.go | 20 +++++++++++--------- internal/router/m2_test.go | 14 ++++++++++++++ internal/store/m2_test.go | 3 +++ 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 19ddfdf..685971c 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -163,6 +163,9 @@ func TestCreateListValidation(t *testing.T) { `{"name":".dev","domain":"example.org"}`, `{"name":"dev.","domain":"example.org"}`, `{"name":"a..b","domain":"example.org"}`, + `{"name":"a{b","domain":"example.org"}`, + `{"name":"","domain":"example.org","title":"Dev"}`, + `{"name":" ","domain":"example.org","title":"Dev"}`, `{"name":"` + strings.Repeat("a", 31) + `","domain":"example.org"}`, // Domains that are not host names. `{"name":"dev","domain":"exa mple.org"}`, @@ -806,3 +809,34 @@ func TestUpdateListTitleUsesTheSameRules(t *testing.T) { } expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"title":"`+strings.Repeat("x", 200)+`"}`, true), 200) } + +func TestValidDomainLengthLimits(t *testing.T) { + l63 := strings.Repeat("a", 63) + d253 := l63 + "." + l63 + "." + l63 + "." + strings.Repeat("a", 61) + if len(d253) != 253 || !validDomain(d253) { + t.Fatalf("a 253-character domain is valid (%d)", len(d253)) + } + if validDomain(d253 + "a") { + t.Fatal("254 characters is too long") + } + if !validDomain(l63+".org") || validDomain("") || validName("") { + t.Fatal("63-character labels are valid, empty names and domains are not") + } +} + +func TestCreateListTrimsTheTitle(t *testing.T) { + h := newServer(t) + rec := do(t, h, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":" Dev team \t"}`, true) + expect(t, rec, 201) + var l listJSON + decode(t, rec, &l) + if l.Title != "Dev team" { + t.Fatalf("title = %q", l.Title) + } + rec = do(t, h, "POST", "/api/v1/lists", `{"name":"ops","domain":"example.org","title":" "}`, true) + expect(t, rec, 201) + decode(t, rec, &l) + if l.Title != "ops" { + t.Fatalf("a blank title falls back to the name: %q", l.Title) + } +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4381983..9a27087 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -262,15 +262,17 @@ func TestBaseURLIsChecked(t *testing.T) { } } for _, bad := range []string{ - "http://lists.example.org", // RFC 8058 needs https - "http://10.0.0.5:8081", // only loopback may use http - "ftp://lists.example.org", // not http(s) at all - "lists.example.org", // no scheme - "//lists.example.org", // no scheme - "https://", // no host - "https:///path", // no host - "https://lists.example.org?x=1", // query - "https://lists.example.org/#top", // fragment + "http://lists.example.org", // RFC 8058 needs https + "http://10.0.0.5:8081", // only loopback may use http + "ftp://lists.example.org", // not http(s) at all + "lists.example.org", // no scheme + "//lists.example.org", // no scheme + "https://", // no host + "https:///path", // no host + "https://lists.example.org?x=1", // query + "https://lists.example.org/#top", + "https://lists.example.org/#", // an empty fragment is still one + "https://lists.example.org/?", // an empty query is still one // fragment "https://user:pw@lists.example.org", // userinfo "https://user@lists.example.org", "https://lists.example.org/a b", diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index 5d934cb..3e611a6 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -571,3 +571,17 @@ func TestConfirmTokensAreUnpredictable(t *testing.T) { seen[tok] = true } } + +// Closing a list to new members must not stop a member from leaving. +func TestLeaveConfirmationWorksOnAClosedList(t *testing.T) { + e := newEnv(t) + e.command("alice@example.com", "dev+leave@example.org", nil) + tok := token(t, lastMail(t, e.out)) + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) + if _, err := e.command("alice@example.com", "dev+confirm-"+tok+"@example.org", nil); err != nil { + t.Fatalf("leave on a closed list: %v", err) + } + if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); ok { + t.Fatal("alice must be gone") + } +} diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 0c71628..79765bf 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -676,6 +676,9 @@ func TestConfirmPendingExpiredIsNotFound(t *testing.T) { if ok, _ := s.IsMember(l.ID, "dave@x.com"); ok { t.Fatal("an expired token must not subscribe") } + if pendingRows(t, s) != 0 { + t.Fatal("an expired token is removed when it is presented") + } } // Two confirmations of one token at the same time: exactly one applies.