feat(web): public subscribe form for lists with a public archive and open joins
- GET /subscribe/{address}: email field, an off-screen honeypot named website,
a Subscribe button; 404 (the usual page) for a private, closed or unknown list
- POST /subscribe/{address}: one plain ASCII address of at most 254 characters,
else the form again (400) with the escaped typed value; a filled honeypot gets
the success page and nothing happens; otherwise Router.RequestJoin and always
the same neutral page, so a member, a rate limited address or an unknown
one cannot be told apart; the body is limited to 4 KiB (413)
- at most 10 POSTs per hour per TCP peer address (X-Forwarded-For is ignored),
then 429 with Retry-After; idle addresses are swept; the clock is injectable
- a Subscribe link on the archive page of a list that qualifies, nothing else
- no-store on the subscribe pages; the CSP is unchanged
- tests for each of these, including escaping, enumeration, the limit and its expiry
This commit is contained in:
parent
daea68f69d
commit
922e2dc500
3 changed files with 569 additions and 5 deletions
81
internal/web/limit.go
Normal file
81
internal/web/limit.go
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
package web
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A person subscribes once; a script posting the form for other people's addresses is what this
|
||||||
|
// slows down: at most subscribeMax POSTs per subscribeWindow from one address. The window starts
|
||||||
|
// at the first request of an address and everything about it is forgotten when it ends.
|
||||||
|
const (
|
||||||
|
subscribeMax = 10
|
||||||
|
subscribeWindow = time.Hour
|
||||||
|
sweepEvery = time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
type visits struct {
|
||||||
|
count int
|
||||||
|
first time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type subscribeLimiter struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
now func() time.Time
|
||||||
|
byIP map[string]*visits
|
||||||
|
lastSweep time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSubscribeLimiter(now func() time.Time) *subscribeLimiter {
|
||||||
|
return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// allow counts a request from ip. When the address is over its limit it returns false and how long
|
||||||
|
// until it may try again; a refused request is not counted.
|
||||||
|
func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
now := l.now()
|
||||||
|
l.sweep(now)
|
||||||
|
v := l.byIP[ip]
|
||||||
|
if v == nil || now.Sub(v.first) >= subscribeWindow {
|
||||||
|
v = &visits{first: now}
|
||||||
|
l.byIP[ip] = v
|
||||||
|
}
|
||||||
|
if v.count >= subscribeMax {
|
||||||
|
return v.first.Add(subscribeWindow).Sub(now), false
|
||||||
|
}
|
||||||
|
v.count++
|
||||||
|
return 0, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// sweep forgets addresses whose window is over, so the map cannot grow forever.
|
||||||
|
func (l *subscribeLimiter) sweep(now time.Time) {
|
||||||
|
if now.Sub(l.lastSweep) < sweepEvery {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.lastSweep = now
|
||||||
|
for ip, v := range l.byIP {
|
||||||
|
if now.Sub(v.first) >= subscribeWindow {
|
||||||
|
delete(l.byIP, ip)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *subscribeLimiter) size() int {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return len(l.byIP)
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientIP is the address of the TCP peer. X-Forwarded-For is deliberately ignored: anyone can
|
||||||
|
// send it, so trusting it would let a script pick a fresh identity for every request.
|
||||||
|
func clientIP(r *http.Request) string {
|
||||||
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||||
|
if err != nil {
|
||||||
|
return r.RemoteAddr
|
||||||
|
}
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
@ -8,10 +8,12 @@ import (
|
||||||
"html/template"
|
"html/template"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/mail"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"codeberg.org/squidd/parsimony/assets"
|
"codeberg.org/squidd/parsimony/assets"
|
||||||
"codeberg.org/squidd/parsimony/internal/archive"
|
"codeberg.org/squidd/parsimony/internal/archive"
|
||||||
|
|
@ -28,11 +30,15 @@ type server struct {
|
||||||
store *store.Store
|
store *store.Store
|
||||||
router *router.Router
|
router *router.Router
|
||||||
tmpl *template.Template
|
tmpl *template.Template
|
||||||
|
limit *subscribeLimiter
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns the public web handler.
|
// New returns the public web handler.
|
||||||
func New(s *store.Store, rt *router.Router) http.Handler {
|
func New(s *store.Store, rt *router.Router) http.Handler { return newHandler(s, rt, time.Now) }
|
||||||
srv := &server{store: s, router: rt, tmpl: template.Must(template.New("").Funcs(template.FuncMap{
|
|
||||||
|
// newHandler is New with an injectable clock, for the tests.
|
||||||
|
func newHandler(s *store.Store, rt *router.Router, now func() time.Time) http.Handler {
|
||||||
|
srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), tmpl: template.Must(template.New("").Funcs(template.FuncMap{
|
||||||
"hide": archive.Obfuscate,
|
"hide": archive.Obfuscate,
|
||||||
// The logo is a trusted static asset: inline, so the strict CSP needs no image source for it.
|
// The logo is a trusted static asset: inline, so the strict CSP needs no image source for it.
|
||||||
"mark": func() template.HTML { return template.HTML(assets.Mark) },
|
"mark": func() template.HTML { return template.HTML(assets.Mark) },
|
||||||
|
|
@ -44,6 +50,8 @@ func New(s *store.Store, rt *router.Router) http.Handler {
|
||||||
mux.HandleFunc("GET /{$}", srv.index)
|
mux.HandleFunc("GET /{$}", srv.index)
|
||||||
mux.HandleFunc("GET /archive/{address}", srv.archive)
|
mux.HandleFunc("GET /archive/{address}", srv.archive)
|
||||||
mux.HandleFunc("GET /archive/{address}/{id}", srv.message)
|
mux.HandleFunc("GET /archive/{address}/{id}", srv.message)
|
||||||
|
mux.HandleFunc("GET /subscribe/{address}", srv.subscribeForm)
|
||||||
|
mux.HandleFunc("POST /subscribe/{address}", srv.subscribe)
|
||||||
mux.HandleFunc("GET /unsubscribe/{token}", srv.unsubscribePage)
|
mux.HandleFunc("GET /unsubscribe/{token}", srv.unsubscribePage)
|
||||||
mux.HandleFunc("POST /unsubscribe/{token}", srv.unsubscribe)
|
mux.HandleFunc("POST /unsubscribe/{token}", srv.unsubscribe)
|
||||||
// Catch-all for GET so unknown paths get the HTML 404 page and other
|
// Catch-all for GET so unknown paths get the HTML 404 page and other
|
||||||
|
|
@ -63,6 +71,9 @@ func secure(next http.Handler) http.Handler {
|
||||||
if strings.HasPrefix(r.URL.Path, "/unsubscribe/") {
|
if strings.HasPrefix(r.URL.Path, "/unsubscribe/") {
|
||||||
h.Set("Cache-Control", "no-store") // the URL carries a secret
|
h.Set("Cache-Control", "no-store") // the URL carries a secret
|
||||||
}
|
}
|
||||||
|
if strings.HasPrefix(r.URL.Path, "/subscribe/") {
|
||||||
|
h.Set("Cache-Control", "no-store") // a form with an address typed into it
|
||||||
|
}
|
||||||
next.ServeHTTP(w, r)
|
next.ServeHTTP(w, r)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -118,6 +129,8 @@ type archiveView struct {
|
||||||
PrevURL string
|
PrevURL string
|
||||||
NextURL string
|
NextURL string
|
||||||
ResultText string
|
ResultText string
|
||||||
|
// CanSubscribe is true when /subscribe/{address} serves this list.
|
||||||
|
CanSubscribe bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *server) archive(w http.ResponseWriter, r *http.Request) {
|
func (s *server) archive(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
@ -152,7 +165,7 @@ func (s *server) archive(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
v := archiveView{List: l, Query: q, Messages: ms, Total: total, Page: page}
|
v := archiveView{List: l, Query: q, Messages: ms, Total: total, Page: page, CanSubscribe: l.JoinPolicy == store.JoinOpen}
|
||||||
link := func(p int) string {
|
link := func(p int) string {
|
||||||
vals := url.Values{}
|
vals := url.Values{}
|
||||||
if q != "" {
|
if q != "" {
|
||||||
|
|
@ -216,6 +229,108 @@ func (s *server) message(w http.ResponseWriter, r *http.Request) {
|
||||||
s.render(w, http.StatusOK, "message", messageView{List: l, Message: m, Thread: thread})
|
s.render(w, http.StatusOK, "message", messageView{List: l, Message: m, Thread: thread})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// subscribableList resolves a list for the subscribe form: it must have a public archive and accept
|
||||||
|
// self-service subscriptions. Anything else is reported as missing, so a private or closed list
|
||||||
|
// cannot be told from one that does not exist.
|
||||||
|
func (s *server) subscribableList(address string) (store.List, bool, error) {
|
||||||
|
l, ok, err := s.publicList(address)
|
||||||
|
if err != nil || !ok || l.JoinPolicy != store.JoinOpen {
|
||||||
|
return store.List{}, false, err
|
||||||
|
}
|
||||||
|
return l, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type subscribeView struct {
|
||||||
|
List store.List
|
||||||
|
Email string
|
||||||
|
Error string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *server) subscribeForm(w http.ResponseWriter, r *http.Request) {
|
||||||
|
l, ok, err := s.subscribableList(r.PathValue("address"))
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
s.fail(w, err)
|
||||||
|
case !ok:
|
||||||
|
s.notFound(w)
|
||||||
|
default:
|
||||||
|
s.render(w, http.StatusOK, "subscribe", subscribeView{List: l})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxSubscribeBody is plenty for an email address and a honeypot field.
|
||||||
|
const maxSubscribeBody = 4 << 10
|
||||||
|
|
||||||
|
func (s *server) subscribe(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if wait, ok := s.limit.allow(clientIP(r)); !ok {
|
||||||
|
w.Header().Set("Retry-After", strconv.Itoa(int((wait+time.Second-1)/time.Second)))
|
||||||
|
s.render(w, http.StatusTooManyRequests, "toomany", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l, ok, err := s.subscribableList(r.PathValue("address"))
|
||||||
|
if err != nil {
|
||||||
|
s.fail(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
s.notFound(w)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, maxSubscribeBody)
|
||||||
|
if err := r.ParseForm(); err != nil {
|
||||||
|
var tooBig *http.MaxBytesError
|
||||||
|
if errors.As(err, &tooBig) {
|
||||||
|
http.Error(w, "request too large", http.StatusRequestEntityTooLarge)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A bot that fills every field gets the page a person gets, and nothing happens.
|
||||||
|
if r.PostForm.Get("website") != "" {
|
||||||
|
s.render(w, http.StatusOK, "subscribed", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
typed := r.PostForm.Get("email")
|
||||||
|
addr, ok := plainAddress(typed)
|
||||||
|
if !ok {
|
||||||
|
s.render(w, http.StatusBadRequest, "subscribe", subscribeView{List: l, Email: printable(typed), Error: "Enter a valid email address."})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.router.RequestJoin(l.Address(), addr); err != nil {
|
||||||
|
s.fail(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.render(w, http.StatusOK, "subscribed", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// printable drops control characters, so what was typed can be shown back safely.
|
||||||
|
func printable(s string) string {
|
||||||
|
return strings.Map(func(r rune) rune {
|
||||||
|
if unicode.IsControl(r) {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// plainAddress accepts one bare ASCII address of at most 254 characters: no display name, no angle
|
||||||
|
// brackets, no list, no quoting, no control characters.
|
||||||
|
func plainAddress(s string) (string, bool) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" || len(s) > 254 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
for _, c := range []byte(s) {
|
||||||
|
if c <= ' ' || c >= 0x7f {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a, err := mail.ParseAddress(s)
|
||||||
|
if err != nil || a.Name != "" || a.Address != s {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return s, true
|
||||||
|
}
|
||||||
|
|
||||||
func (s *server) unsubscribePage(w http.ResponseWriter, r *http.Request) {
|
func (s *server) unsubscribePage(w http.ResponseWriter, r *http.Request) {
|
||||||
l, err := s.router.CheckUnsubscribe(r.PathValue("token"))
|
l, err := s.router.CheckUnsubscribe(r.PathValue("token"))
|
||||||
switch {
|
switch {
|
||||||
|
|
@ -259,6 +374,9 @@ pre{white-space:pre-wrap;word-wrap:break-word;background:#f6f6f6;padding:1rem;bo
|
||||||
input[type=search]{padding:.35rem;width:16rem}
|
input[type=search]{padding:.35rem;width:16rem}
|
||||||
button{padding:.5rem 1rem;font-size:1rem}
|
button{padding:.5rem 1rem;font-size:1rem}
|
||||||
nav a{margin-right:1rem}
|
nav a{margin-right:1rem}
|
||||||
|
.error{color:#b00020}
|
||||||
|
.hp{position:absolute;left:-10000px;width:1px;height:1px;overflow:hidden}
|
||||||
|
input[type=email]{padding:.35rem;width:20rem;max-width:100%}
|
||||||
.brand{display:inline-block;margin-bottom:1rem}
|
.brand{display:inline-block;margin-bottom:1rem}
|
||||||
.brand svg{width:2rem;height:2rem;display:block}
|
.brand svg{width:2rem;height:2rem;display:block}
|
||||||
</style></head><body>
|
</style></head><body>
|
||||||
|
|
@ -272,7 +390,7 @@ nav a{margin-right:1rem}
|
||||||
{{template "foot"}}{{end}}
|
{{template "foot"}}{{end}}
|
||||||
|
|
||||||
{{define "archive"}}{{template "head" .List.Title}}
|
{{define "archive"}}{{template "head" .List.Title}}
|
||||||
<nav><a href="/">All lists</a></nav>
|
<nav><a href="/">All lists</a>{{if .CanSubscribe}}<a href="/subscribe/{{.List.Address}}">Subscribe</a>{{end}}</nav>
|
||||||
<h1>{{.List.Title}}</h1>
|
<h1>{{.List.Title}}</h1>
|
||||||
<p class="muted">{{.List.Address}}</p>
|
<p class="muted">{{.List.Address}}</p>
|
||||||
<form method="get" action="/archive/{{.List.Address}}"><input type="search" name="q" value="{{.Query}}" placeholder="Search"> <button>Search</button></form>
|
<form method="get" action="/archive/{{.List.Address}}"><input type="search" name="q" value="{{.Query}}" placeholder="Search"> <button>Search</button></form>
|
||||||
|
|
@ -294,6 +412,28 @@ nav a{margin-right:1rem}
|
||||||
<span class="muted">{{hide .Sender}}, {{date .Date}}</span></li>{{end}}</ul>{{end}}
|
<span class="muted">{{hide .Sender}}, {{date .Date}}</span></li>{{end}}</ul>{{end}}
|
||||||
{{template "foot"}}{{end}}
|
{{template "foot"}}{{end}}
|
||||||
|
|
||||||
|
{{define "subscribe"}}{{template "head" (print "Subscribe to " .List.Title)}}
|
||||||
|
<nav><a href="/">All lists</a><a href="/archive/{{.List.Address}}">{{.List.Title}}</a></nav>
|
||||||
|
<h1>Subscribe to {{.List.Title}}</h1>
|
||||||
|
<p class="muted">{{.List.Address}}</p>
|
||||||
|
{{if .Error}}<p class="error" role="alert">{{.Error}}</p>{{end}}
|
||||||
|
<form method="post" action="/subscribe/{{.List.Address}}">
|
||||||
|
<p><label>Email address<br><input type="email" name="email" value="{{.Email}}" required maxlength="254" autocomplete="email"></label></p>
|
||||||
|
<div class="hp" aria-hidden="true"><label>Leave this empty<input type="text" name="website" tabindex="-1" autocomplete="off"></label></div>
|
||||||
|
<button>Subscribe</button>
|
||||||
|
</form>
|
||||||
|
<p class="muted">We send one message to confirm the address. Nothing happens until you reply to it.</p>
|
||||||
|
{{template "foot"}}{{end}}
|
||||||
|
|
||||||
|
{{define "subscribed"}}{{template "head" "Check your email"}}
|
||||||
|
<h1>Check your email</h1>
|
||||||
|
<p>If this address can subscribe, a confirmation message is on its way. Open it and reply to confirm.</p>
|
||||||
|
{{template "foot"}}{{end}}
|
||||||
|
|
||||||
|
{{define "toomany"}}{{template "head" "Too many requests"}}
|
||||||
|
<h1>Too many requests</h1><p>Please try again later.</p>
|
||||||
|
{{template "foot"}}{{end}}
|
||||||
|
|
||||||
{{define "unsubscribe"}}{{template "head" "Unsubscribe"}}
|
{{define "unsubscribe"}}{{template "head" "Unsubscribe"}}
|
||||||
<h1>Unsubscribe</h1>
|
<h1>Unsubscribe</h1>
|
||||||
<p>Stop receiving messages from <strong>{{.Title}}</strong> ({{.Address}})?</p>
|
<p>Stop receiving messages from <strong>{{.Title}}</strong> ({{.Address}})?</p>
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,21 @@
|
||||||
package web
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
"html"
|
"html"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"codeberg.org/squidd/parsimony/assets"
|
"codeberg.org/squidd/parsimony/assets"
|
||||||
|
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||||
|
"codeberg.org/squidd/parsimony/internal/queue"
|
||||||
"codeberg.org/squidd/parsimony/internal/router"
|
"codeberg.org/squidd/parsimony/internal/router"
|
||||||
"codeberg.org/squidd/parsimony/internal/store"
|
"codeberg.org/squidd/parsimony/internal/store"
|
||||||
)
|
)
|
||||||
|
|
@ -23,6 +27,8 @@ type fixture struct {
|
||||||
pub store.List
|
pub store.List
|
||||||
alice int64
|
alice int64
|
||||||
bobTok string
|
bobTok string
|
||||||
|
out *outbound.Fake
|
||||||
|
clock time.Time // what the handler thinks the time is; tests move it
|
||||||
}
|
}
|
||||||
|
|
||||||
func newFixture(t *testing.T) *fixture {
|
func newFixture(t *testing.T) *fixture {
|
||||||
|
|
@ -38,12 +44,32 @@ func newFixture(t *testing.T) *fixture {
|
||||||
s.SetPolicies(priv.ID, store.PostMembers, store.JoinOpen, store.ArchivePrivate)
|
s.SetPolicies(priv.ID, store.PostMembers, store.JoinOpen, store.ArchivePrivate)
|
||||||
off, _ := s.CreateList("off", "example.org", "Off")
|
off, _ := s.CreateList("off", "example.org", "Off")
|
||||||
s.SetPolicies(off.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff)
|
s.SetPolicies(off.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff)
|
||||||
|
closed, _ := s.CreateList("closed", "example.org", "Closed")
|
||||||
|
s.SetPolicies(closed.ID, store.PostMembers, store.JoinClosed, store.ArchivePublic)
|
||||||
|
|
||||||
s.AddMember(pub.ID, "alice@example.com", "Alice")
|
s.AddMember(pub.ID, "alice@example.com", "Alice")
|
||||||
s.AddMember(pub.ID, "bob@example.com", "Bob")
|
s.AddMember(pub.ID, "bob@example.com", "Bob")
|
||||||
ms, _ := s.Members(pub.ID)
|
ms, _ := s.Members(pub.ID)
|
||||||
rt := router.New(s)
|
rt := router.New(s)
|
||||||
return &fixture{h: New(s, rt), s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID)}
|
f := &fixture{s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID),
|
||||||
|
out: &outbound.Fake{}, clock: time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)}
|
||||||
|
f.h = newHandler(s, rt, func() time.Time { return f.clock })
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// flush sends what is queued, as the background worker would, and returns everything sent so far.
|
||||||
|
func (f *fixture) flush(t *testing.T) []outbound.Sent {
|
||||||
|
t.Helper()
|
||||||
|
w := &queue.Worker{Store: f.s, Sender: f.out}
|
||||||
|
for {
|
||||||
|
st, err := w.RunOnce(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if st == (queue.Stats{}) {
|
||||||
|
return f.out.Sent
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fixture) add(t *testing.T, listID int64, id, subject, sender, body string, at int64, inReplyTo string) int64 {
|
func (f *fixture) add(t *testing.T, listID int64, id, subject, sender, body string, at int64, inReplyTo string) int64 {
|
||||||
|
|
@ -324,3 +350,320 @@ func TestTheCSPAllowsOnlyDataImages(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Subscribe form.
|
||||||
|
|
||||||
|
const neutralText = "If this address can subscribe, a confirmation message is on its way. Open it and reply to confirm."
|
||||||
|
|
||||||
|
// subscribe posts a form from the TCP peer ip.
|
||||||
|
func subscribe(t *testing.T, f *fixture, list, ip string, form url.Values) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
return subscribeRaw(f.h, list, ip, form.Encode(), nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func subscribeRaw(h http.Handler, list, ip, body string, hdr map[string]string) *httptest.ResponseRecorder {
|
||||||
|
req := httptest.NewRequest("POST", "/subscribe/"+list, strings.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
for k, v := range hdr {
|
||||||
|
req.Header.Set(k, v)
|
||||||
|
}
|
||||||
|
req.RemoteAddr = ip + ":4321"
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func email(addr string) url.Values { return url.Values{"email": {addr}} }
|
||||||
|
|
||||||
|
func TestSubscribeFormIsShownForPublicOpenLists(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
rec := get(t, f.h, "/subscribe/dev@example.org")
|
||||||
|
wantStatus(t, rec, 200)
|
||||||
|
wantBody(t, rec, `method="post"`, `action="/subscribe/dev@example.org"`, `name="email"`, `type="email"`,
|
||||||
|
`name="website"`, "Dev <b>talk</b>", "<button>Subscribe</button>")
|
||||||
|
wantNoBody(t, rec, "<b>talk</b>")
|
||||||
|
if got := rec.Header().Get("Cache-Control"); got != "no-store" {
|
||||||
|
t.Errorf("Cache-Control = %q, want no-store", got)
|
||||||
|
}
|
||||||
|
if csp := rec.Header().Get("Content-Security-Policy"); csp != "default-src 'none'; img-src data:; style-src 'unsafe-inline'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'" {
|
||||||
|
t.Errorf("the CSP must not change: %q", csp)
|
||||||
|
}
|
||||||
|
// The honeypot is hidden from people and from assistive technology, and nothing fills it in.
|
||||||
|
wantBody(t, rec, `class="hp"`, `tabindex="-1"`, `autocomplete="off"`, `aria-hidden="true"`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeIs404ForEverythingElseAndLooksLikeAMissingList(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
missing := get(t, f.h, "/subscribe/ghost@example.org")
|
||||||
|
wantStatus(t, missing, 404)
|
||||||
|
for _, p := range []string{
|
||||||
|
"/subscribe/secret@example.org", // private archive
|
||||||
|
"/subscribe/off@example.org", // no archive
|
||||||
|
"/subscribe/closed@example.org", // public archive, nobody may join
|
||||||
|
"/subscribe/not-an-address",
|
||||||
|
"/subscribe/..%2Fdev@example.org",
|
||||||
|
"/subscribe/dev@example.org%2F..%2Fsecret@example.org",
|
||||||
|
"/subscribe/dev@example.org%00",
|
||||||
|
"/subscribe/dev@example.org/extra",
|
||||||
|
"/subscribe/",
|
||||||
|
"/subscribe",
|
||||||
|
} {
|
||||||
|
rec := get(t, f.h, p)
|
||||||
|
wantStatus(t, rec, 404)
|
||||||
|
if rec.Body.String() != missing.Body.String() {
|
||||||
|
t.Errorf("%s: the 404 page must be the same as for a missing list", p)
|
||||||
|
}
|
||||||
|
rec = request(t, f.h, "POST", p)
|
||||||
|
if rec.Code != 404 && rec.Code != 405 {
|
||||||
|
t.Errorf("POST %s = %d, want 404 or 405", p, rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeQueuesOneConfirmationMail(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
rec := subscribe(t, f, "dev@example.org", "192.0.2.1", email("carol@x.com"))
|
||||||
|
wantStatus(t, rec, 200)
|
||||||
|
wantBody(t, rec, neutralText)
|
||||||
|
wantNoBody(t, rec, "carol")
|
||||||
|
if got := rec.Header().Get("Cache-Control"); got != "no-store" {
|
||||||
|
t.Errorf("Cache-Control = %q, want no-store", got)
|
||||||
|
}
|
||||||
|
sent := f.flush(t)
|
||||||
|
if len(sent) != 1 || len(sent[0].To) != 1 || sent[0].To[0] != "carol@x.com" {
|
||||||
|
t.Fatalf("want one confirmation mail to carol, got %+v", sent)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(sent[0].Data), "Reply-To: dev+confirm-") {
|
||||||
|
t.Fatalf("not a confirmation mail: %s", sent[0].Data)
|
||||||
|
}
|
||||||
|
if ok, _ := f.s.IsMember(f.pub.ID, "carol@x.com"); ok {
|
||||||
|
t.Fatal("not a member before confirming")
|
||||||
|
}
|
||||||
|
// Exactly one pending confirmation: the store refuses a second one within the hour.
|
||||||
|
subscribe(t, f, "dev@example.org", "192.0.2.1", email("carol@x.com"))
|
||||||
|
if sent := f.flush(t); len(sent) != 1 {
|
||||||
|
t.Fatalf("a repeat must not queue another mail, sent %d", len(sent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeAnswersTheSameWhateverHappens(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
f.s.AddMember(f.pub.ID, "member@x.com", "")
|
||||||
|
fresh := subscribe(t, f, "dev@example.org", "192.0.2.1", email("new@x.com"))
|
||||||
|
member := subscribe(t, f, "dev@example.org", "192.0.2.2", email("member@x.com"))
|
||||||
|
limited := subscribe(t, f, "dev@example.org", "192.0.2.3", email("new@x.com")) // the store rate limits it
|
||||||
|
if fresh.Code != 200 || member.Code != 200 || limited.Code != 200 {
|
||||||
|
t.Fatalf("statuses %d %d %d", fresh.Code, member.Code, limited.Code)
|
||||||
|
}
|
||||||
|
for name, rec := range map[string]*httptest.ResponseRecorder{"member": member, "rate limited": limited} {
|
||||||
|
if rec.Body.String() != fresh.Body.String() {
|
||||||
|
t.Errorf("%s: the page differs from the page for a new address", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantBody(t, fresh, neutralText)
|
||||||
|
if sent := f.flush(t); len(sent) != 1 || sent[0].To[0] != "new@x.com" {
|
||||||
|
t.Fatalf("only the new address gets mail: %+v", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeAcceptsTheAddressInAnyCase(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
f.s.AddMember(f.pub.ID, "member@x.com", "")
|
||||||
|
subscribe(t, f, "dev@example.org", "192.0.2.1", email(" MEMBER@X.com "))
|
||||||
|
if sent := f.flush(t); len(sent) != 0 {
|
||||||
|
t.Fatalf("a member gets no mail whatever the case: %+v", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeHoneypotDoesNothing(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
good := subscribe(t, f, "dev@example.org", "192.0.2.1", email("ok@x.com"))
|
||||||
|
f.flush(t)
|
||||||
|
f.out.Sent = nil
|
||||||
|
bot := subscribe(t, f, "dev@example.org", "192.0.2.2", url.Values{"email": {"bot@x.com"}, "website": {"http://spam"}})
|
||||||
|
wantStatus(t, bot, 200)
|
||||||
|
if bot.Body.String() != good.Body.String() {
|
||||||
|
t.Error("a bot must see the page a person sees")
|
||||||
|
}
|
||||||
|
// Even a bad address gets the success page when the honeypot is filled.
|
||||||
|
bot = subscribe(t, f, "dev@example.org", "192.0.2.2", url.Values{"email": {"nonsense"}, "website": {"x"}})
|
||||||
|
wantStatus(t, bot, 200)
|
||||||
|
wantBody(t, bot, neutralText)
|
||||||
|
if sent := f.flush(t); len(sent) != 0 {
|
||||||
|
t.Fatalf("the honeypot must queue nothing: %+v", sent)
|
||||||
|
}
|
||||||
|
// No pending confirmation was created either: a real request for the same address still works.
|
||||||
|
subscribe(t, f, "dev@example.org", "192.0.2.3", email("bot@x.com"))
|
||||||
|
if sent := f.flush(t); len(sent) != 1 {
|
||||||
|
t.Fatalf("a real request after the honeypot must send its mail, sent %d", len(sent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeRejectsBadAddressesAndEscapesWhatWasTyped(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
long := strings.Repeat("a", 250) + "@x.com" // 256 characters
|
||||||
|
for i, c := range []struct{ name, in string }{
|
||||||
|
{"empty", ""},
|
||||||
|
{"blank", " "},
|
||||||
|
{"no at sign", "carol"},
|
||||||
|
{"display name", "Carol <carol@x.com>"},
|
||||||
|
{"angle brackets only", "<carol@x.com>"},
|
||||||
|
{"two addresses", "a@x.com, b@x.com"},
|
||||||
|
{"two addresses with a space", "a@x.com b@x.com"},
|
||||||
|
{"markup", `"><script>alert(1)</script>`},
|
||||||
|
{"markup in an address", `<script>alert(1)</script>@x.com`},
|
||||||
|
{"quoted local part", `"a b"@x.com`},
|
||||||
|
{"control character", "a\x01@x.com"},
|
||||||
|
{"line break", "a@x.com\r\nBcc: v@x.com"},
|
||||||
|
{"non-ASCII", "cärol@x.com"},
|
||||||
|
{"too long", long},
|
||||||
|
} {
|
||||||
|
rec := subscribe(t, f, "dev@example.org", fmt.Sprintf("192.0.2.%d", i+1), email(c.in)) // one IP each: the limit is per IP
|
||||||
|
if rec.Code != 400 {
|
||||||
|
t.Errorf("%s: status = %d, want 400", c.name, rec.Code)
|
||||||
|
}
|
||||||
|
wantBody(t, rec, `name="email"`, "valid email address")
|
||||||
|
wantNoBody(t, rec, neutralText, "<script>", "\x01")
|
||||||
|
}
|
||||||
|
if sent := f.flush(t); len(sent) != 0 {
|
||||||
|
t.Fatalf("no mail for a bad address: %+v", sent)
|
||||||
|
}
|
||||||
|
// The typed value comes back, escaped.
|
||||||
|
rec := subscribe(t, f, "dev@example.org", "192.0.2.100", email(`<script>alert(1)</script>`))
|
||||||
|
wantBody(t, rec, `value="<script>alert(1)</script>"`)
|
||||||
|
// A 254-character address is the longest accepted.
|
||||||
|
ok := strings.Repeat("a", 64) + "@" + strings.Repeat("b", 63) + "." + strings.Repeat("c", 63) + "." + strings.Repeat("d", 57) + ".com"
|
||||||
|
if len(ok) != 254 {
|
||||||
|
t.Fatalf("setup: %d", len(ok))
|
||||||
|
}
|
||||||
|
wantBody(t, subscribe(t, f, "dev@example.org", "192.0.2.101", email(ok)), neutralText)
|
||||||
|
wantBody(t, subscribe(t, f, "dev@example.org", "192.0.2.101", email("a"+ok)), "valid email address")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeBodyIsLimitedTo4KiB(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
body := func(n int) string {
|
||||||
|
head := "email=carol%40x.com&pad="
|
||||||
|
return head + strings.Repeat("x", n-len(head))
|
||||||
|
}
|
||||||
|
rec := subscribeRaw(f.h, "dev@example.org", "192.0.2.1", body(4096), nil)
|
||||||
|
wantStatus(t, rec, 200)
|
||||||
|
wantBody(t, rec, neutralText)
|
||||||
|
rec = subscribeRaw(f.h, "dev@example.org", "192.0.2.1", body(4097), nil)
|
||||||
|
wantStatus(t, rec, http.StatusRequestEntityTooLarge)
|
||||||
|
wantNoBody(t, rec, neutralText)
|
||||||
|
rec = subscribeRaw(f.h, "dev@example.org", "192.0.2.1", body(1<<20), nil)
|
||||||
|
wantStatus(t, rec, http.StatusRequestEntityTooLarge)
|
||||||
|
if sent := f.flush(t); len(sent) != 1 {
|
||||||
|
t.Fatalf("only the small request sends mail, sent %d", len(sent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeNeverMailsTheAddressForAClosedOrHiddenList(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
for _, l := range []string{"closed@example.org", "secret@example.org", "off@example.org", "ghost@example.org"} {
|
||||||
|
rec := subscribe(t, f, l, "192.0.2.1", email("victim@x.com"))
|
||||||
|
wantStatus(t, rec, 404)
|
||||||
|
wantNoBody(t, rec, neutralText)
|
||||||
|
}
|
||||||
|
if sent := f.flush(t); len(sent) != 0 {
|
||||||
|
t.Fatalf("no mail for these lists: %+v", sent)
|
||||||
|
}
|
||||||
|
// Even after the list opens, nothing was left pending.
|
||||||
|
cl, _ := f.s.FindList("closed@example.org")
|
||||||
|
f.s.SetPolicies(cl.ID, store.PostMembers, store.JoinOpen, store.ArchivePublic)
|
||||||
|
subscribe(t, f, "closed@example.org", "192.0.2.1", email("victim@x.com"))
|
||||||
|
if sent := f.flush(t); len(sent) != 1 {
|
||||||
|
t.Fatalf("one mail once open, sent %d", len(sent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeIsLimitedPerIPAndTheLimitExpires(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email(fmt.Sprintf("u%d@x.com", i))), 200)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(10 * time.Minute)
|
||||||
|
rec := subscribe(t, f, "dev@example.org", "198.51.100.7", email("u10@x.com"))
|
||||||
|
wantStatus(t, rec, 429)
|
||||||
|
if got := rec.Header().Get("Retry-After"); got != strconv.Itoa(50*60) {
|
||||||
|
t.Errorf("Retry-After = %q, want %d", got, 50*60)
|
||||||
|
}
|
||||||
|
wantNoBody(t, rec, neutralText)
|
||||||
|
// A refused request is not counted again, and queued nothing.
|
||||||
|
if sent := f.flush(t); len(sent) != 10 {
|
||||||
|
t.Fatalf("sent %d, want 10", len(sent))
|
||||||
|
}
|
||||||
|
// Another address is not affected.
|
||||||
|
wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.8", email("other@x.com")), 200)
|
||||||
|
// The window is an hour from the first request.
|
||||||
|
f.clock = f.clock.Add(49*time.Minute + 59*time.Second)
|
||||||
|
wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email("u11@x.com")), 429)
|
||||||
|
f.clock = f.clock.Add(2 * time.Second)
|
||||||
|
wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email("u11@x.com")), 200)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeLimitCountsEveryPostOfTheAddress(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
// Honeypot hits, bad input and unknown lists all count, so they cannot be used to probe for free.
|
||||||
|
forms := []url.Values{{"email": {"x"}}, {"email": {"a@x.com"}, "website": {"x"}}, email("b@x.com")}
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
subscribe(t, f, []string{"dev@example.org", "ghost@example.org"}[i%2], "198.51.100.9", forms[i%3])
|
||||||
|
}
|
||||||
|
wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.9", email("c@x.com")), 429)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeLimitIgnoresForwardedFor(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("203.0.113.%d", i), "X-Real-IP": fmt.Sprintf("203.0.113.%d", i)}
|
||||||
|
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 200)
|
||||||
|
}
|
||||||
|
hdr := map[string]string{"X-Forwarded-For": "203.0.113.200", "X-Real-IP": "203.0.113.200", "Forwarded": "for=203.0.113.201"}
|
||||||
|
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 429)
|
||||||
|
// And naming the blocked address in the header does not hurt somebody else.
|
||||||
|
hdr = map[string]string{"X-Forwarded-For": "198.51.100.7"}
|
||||||
|
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.99", email("a@x.com").Encode(), hdr), 200)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeLimiterForgetsIdleAddresses(t *testing.T) {
|
||||||
|
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
l := newSubscribeLimiter(func() time.Time { return now })
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
l.allow(fmt.Sprintf("192.0.2.%d", i))
|
||||||
|
}
|
||||||
|
if l.size() != 50 {
|
||||||
|
t.Fatalf("size = %d, want 50", l.size())
|
||||||
|
}
|
||||||
|
now = now.Add(30 * time.Minute)
|
||||||
|
l.allow("198.51.100.1")
|
||||||
|
if l.size() != 51 {
|
||||||
|
t.Fatalf("addresses inside the window are kept, size = %d", l.size())
|
||||||
|
}
|
||||||
|
now = now.Add(31 * time.Minute) // the first 50 are now older than an hour
|
||||||
|
l.allow("198.51.100.2")
|
||||||
|
if l.size() != 2 {
|
||||||
|
t.Fatalf("idle addresses must be forgotten, size = %d, want 2", l.size())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeFailsWhenTheDatabaseDoes(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
f.s.Close()
|
||||||
|
wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.1", email("a@x.com")), 500)
|
||||||
|
wantStatus(t, get(t, f.h, "/subscribe/dev@example.org"), 500)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubscribeLinkOnlyOnTheArchivePageOfAQualifyingList(t *testing.T) {
|
||||||
|
f := newFixture(t)
|
||||||
|
link := `href="/subscribe/dev@example.org"`
|
||||||
|
rec := get(t, f.h, "/archive/dev@example.org")
|
||||||
|
wantBody(t, rec, link, ">Subscribe</a>")
|
||||||
|
// A public archive whose list is closed offers none.
|
||||||
|
wantNoBody(t, get(t, f.h, "/archive/closed@example.org"), "/subscribe/")
|
||||||
|
// Nowhere else.
|
||||||
|
id := f.add(t, f.pub.ID, "<1@x>", "Topic", "Alice", "text", 1_700_000_000, "")
|
||||||
|
wantNoBody(t, get(t, f.h, "/"), "/subscribe/")
|
||||||
|
wantNoBody(t, get(t, f.h, fmt.Sprintf("/archive/dev@example.org/%d", id)), "/subscribe/")
|
||||||
|
wantNoBody(t, get(t, f.h, "/unsubscribe/"+f.bobTok), "/subscribe/")
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue