Merge branch 'fix/hygiene-cli-ui' into fix/fable-mediums

# Conflicts:
#	internal/router/m2_test.go
This commit is contained in:
tgiachi 2026-11-10 12:01:20 +01:00
commit b4f45035bb
12 changed files with 617 additions and 43 deletions

View file

@ -78,6 +78,10 @@ parsimony help
`<list>` is an address or a numeric id. Destructive commands (`list rm`, `archive rm`) need `--yes`.
Exit status: 0 success, 1 failure (including lines that failed in `member import`), 2 usage error.
The client refuses to send the token over plain `http://` to any host other than `localhost`, `127.0.0.1` or `::1`
(usage error, exit 2): put the API behind HTTPS, or pass `--insecure` if you really mean it.
Subjects, senders, message bodies and error texts that come from mail are printed with control characters
(escape sequences) shown as `\x1b`-style text instead of being sent to the terminal; `--json` output is already escaped.
`parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works.
In the container: `docker exec parsimony /parsimony list ls`.

View file

@ -76,15 +76,21 @@ describe('held tab', () => {
expect(await screen.findByText(/nothing is waiting/i)).toBeInTheDocument()
})
it('explains how messages get held when posting is not moderated', async () => {
await setup([], aList({ id: 1, post_policy: 'members' }))
expect(await screen.findByText(/set posting to moderated/i)).toBeInTheDocument()
it('explains both ways a message gets held, whatever the posting policy', async () => {
for (const post_policy of ['members', 'moderated'] as const) {
const { unmount } = await setup([], aList({ id: 1, post_policy }))
const hint = await screen.findByText(/messages are held when/i)
expect(hint).toHaveTextContent(/posting is set to moderated/i)
expect(hint).toHaveTextContent(/from header does not match the sender/i)
expect(hint).not.toHaveTextContent(/only/i)
unmount()
}
})
it('does not nag about the policy when the list is moderated', async () => {
await setup([])
await screen.findByText(/nothing is waiting/i)
expect(screen.queryByText(/set posting to moderated/i)).not.toBeInTheDocument()
it('shows no hint when there are held messages', async () => {
await setup([held(1)])
await screen.findByText('Subject 1')
expect(screen.queryByText(/messages are held when/i)).not.toBeInTheDocument()
})
it('approves a message: it is sent and leaves the queue', async () => {

View file

@ -51,9 +51,10 @@ export function HeldTab() {
{held.data && items.length === 0 && (
<div className="space-y-1 text-muted-foreground">
<p>Nothing is waiting for approval.</p>
{list.post_policy !== 'moderated' && (
<p className="text-sm">Messages are held only when you set Posting to moderated in Settings.</p>
)}
<p className="text-sm">
Messages are held when Posting is set to moderated in Settings, or when the From header does not match
the sender.
</p>
</div>
)}
{items.length > 0 && (

View file

@ -8,6 +8,7 @@ import (
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strconv"
@ -22,7 +23,7 @@ type IO struct {
Out, Err io.Writer
}
const usageText = `Usage: parsimony [--url URL] [--json] <command> [arguments]
const usageText = `Usage: parsimony [--url URL] [--insecure] [--json] <command> [arguments]
Administration (talks to the REST API):
list ls show all lists
@ -69,6 +70,7 @@ Environment:
PARSIMONY_URL API address, default http://127.0.0.1:8080 (--url overrides)
--json prints the API's JSON instead of tables.
--insecure allows sending the token over plain http to a host other than localhost.
Exit status: 0 success, 1 failure, 2 usage error.
`
@ -79,6 +81,7 @@ func Run(args []string, io IO, getenv func(string) string) int {
fs := newFlagSet("parsimony")
fs.StringVar(&c.url, "url", "", "")
fs.BoolVar(&c.json, "json", false, "")
fs.BoolVar(&c.insecure, "insecure", false, "")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
fmt.Fprint(io.Out, usageText)
@ -88,7 +91,7 @@ func Run(args []string, io IO, getenv func(string) string) int {
}
rest := fs.Args()
if len(rest) == 0 {
fmt.Fprint(io.Err, "usage: parsimony [--url URL] [--json] <command> (see parsimony help)\n")
fmt.Fprint(io.Err, "usage: parsimony [--url URL] [--insecure] [--json] <command> (see parsimony help)\n")
return 2
}
if rest[0] == "help" {
@ -124,16 +127,27 @@ type cmd struct {
getenv func(string) string
url string
json bool
cl *client
// insecure allows the token to travel over plain http to a non-loopback host.
insecure bool
cl *client
}
// usageErr is an error that is the caller's mistake: exit status 2.
type usageErr string
func (e usageErr) Error() string { return string(e) }
func (c *cmd) usageError(format string, a ...any) int {
fmt.Fprintf(c.io.Err, "error: "+format+"\n", a...)
return 2
}
func (c *cmd) fail(err error) int {
fmt.Fprintf(c.io.Err, "error: %v\n", err)
fmt.Fprintf(c.io.Err, "error: %s\n", printable(err.Error()))
var u usageErr
if errors.As(err, &u) {
return 2
}
return 1
}
@ -153,9 +167,14 @@ func (c *cmd) client() (*client, error) {
if base == "" {
base = "http://127.0.0.1:8080"
}
if _, err := url.Parse(base); err != nil {
u, err := url.Parse(base)
if err != nil {
return nil, fmt.Errorf("invalid URL %q", base)
}
// url.Parse lowercases the scheme.
if u.Scheme == "http" && !isLoopback(u.Hostname()) && !c.insecure {
return nil, usageErr(fmt.Sprintf("refusing to send the token over plain http to %s: use https, or pass --insecure", u.Host))
}
c.cl = &client{
base: strings.TrimRight(base, "/"), token: token,
http: &http.Client{Timeout: 30 * time.Second},
@ -312,10 +331,44 @@ func (c *cmd) printJSON(raw json.RawMessage) int {
return 0
}
// isLoopback reports whether host is this machine: localhost or a loopback address.
func isLoopback(host string) bool {
if strings.EqualFold(strings.TrimSuffix(host, "."), "localhost") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
// printable makes text that came from mail or from the API safe for a terminal: control
// characters (C0, DEL, C1) other than tab and newline are shown as \xNN / \uNNNN instead of
// being sent, so an escape sequence in a subject cannot drive the terminal. CRLF counts as a newline.
func printable(s string) string {
s = strings.ReplaceAll(s, "\r\n", "\n")
var b strings.Builder
for _, r := range s {
switch {
case r == '\t' || r == '\n':
b.WriteRune(r)
case r < 0x20 || r == 0x7f:
fmt.Fprintf(&b, `\x%02x`, r)
case r >= 0x80 && r <= 0x9f:
fmt.Fprintf(&b, `\u%04x`, r)
default:
b.WriteRune(r)
}
}
return b.String()
}
func (c *cmd) table(rows [][]string) {
w := tabwriter.NewWriter(c.io.Out, 0, 8, 2, ' ', 0)
for _, r := range rows {
fmt.Fprintln(w, strings.Join(r, "\t"))
cells := make([]string, len(r))
for i, cell := range r {
cells[i] = printable(cell)
}
fmt.Fprintln(w, strings.Join(cells, "\t"))
}
w.Flush()
}

View file

@ -824,3 +824,172 @@ func TestHeldListingWarnsWhenTheHeaderDiffersFromTheSender(t *testing.T) {
t.Fatalf("no usable From is flagged too: %q", lines["no from"])
}
}
const esc = "\x1b[31mRED\x1b]0;pwned\x07"
// noControl fails when s still holds a control character other than tab and newline.
func noControl(t *testing.T, what, s string) {
t.Helper()
for _, r := range s {
if (r < 0x20 && r != '\t' && r != '\n') || (r >= 0x7f && r <= 0x9f) {
t.Fatalf("%s prints control character %U:\n%q", what, r, s)
}
}
}
func TestPrintableStripsControlCharactersButKeepsTabAndNewline(t *testing.T) {
for in, want := range map[string]string{
"plain text, accents: àèì 日本": "plain text, accents: àèì 日本",
"a\tb\nc": "a\tb\nc",
"a\r\nb": "a\nb",
"\x1b[31mred\x1b[0m": `\x1b[31mred\x1b[0m`,
"us\x1f": `us\x1f`,
"bell\x07 nul\x00 del\x7f": `bell\x07 nul\x00 del\x7f`,
"c1 \u009b31m and \u0085": `c1 \u009b31m and \u0085`,
"c1 edges \u0080 \u009f, kept: ~ \u00a0 \u00a1": `c1 edges \u0080 \u009f, kept: ~ ` + "\u00a0 \u00a1",
"lone\rcarriage": `lone\x0dcarriage`,
"": "",
} {
if got := printable(in); got != want {
t.Errorf("printable(%q) = %q, want %q", in, got, want)
}
}
}
func TestControlCharactersFromMailNeverReachTheTerminal(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
l, _ := e.s.FindList("dev@example.org")
e.s.ArchiveMessage(l.ID, store.Message{MessageID: "<e@x>", Subject: "subj " + esc, Sender: "from " + esc,
Date: time.Unix(1_700_000_000, 0), Body: "line one\r\nbody " + esc + "\r\nlast"})
e.s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob "+esc+" <bob@x.com>\r\nSubject: held "+esc+"\r\n\r\nb"), "reason "+esc)
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "lost@x.com"}})
due, _ := e.s.DueOutbox(10)
e.s.FailOutbox(due[0].ID, "refused: 554 "+esc)
for _, args := range [][]string{
{"archive", "ls", "dev@example.org"}, {"archive", "ls", "dev@example.org", "--q", "subj"},
{"archive", "show", "1"}, {"held", "ls", "dev@example.org"}, {"queue", "ls"},
} {
out := e.ok(t, args...)
noControl(t, strings.Join(args, " "), out)
if !strings.Contains(out, `\x1b[31mRED`) {
t.Errorf("%v: the escape sequence should stay visible as text:\n%s", args, out)
}
}
if out := e.ok(t, "archive", "show", "1"); !strings.Contains(out, "\nbody ") || strings.Contains(out, `\x0d`) {
t.Errorf("CRLF line ends are not control noise: %q", out)
}
}
func TestControlCharactersInListAndMemberNamesAreEscaped(t *testing.T) {
e := newEnv(t)
addr := "a\x1b[31mb\x07@x.org"
l, err := e.s.CreateList("a\x1b[31mb\x07", "x.org", "title "+esc)
if err != nil {
t.Fatal(err)
}
e.s.AddMember(l.ID, "m\x1b[2Jm@x.org", "name "+esc)
mbox := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"
for _, c := range []struct {
stdin string
args []string
}{
{"", []string{"list", "ls"}}, {"", []string{"list", "show", addr}},
{"", []string{"list", "set", addr, "--title", "t"}},
{"", []string{"member", "ls", addr}}, {"", []string{"member", "ls", addr, "--emails"}},
{"", []string{"member", "add", addr, "new@x.com"}},
{"new2@x.com\n", []string{"member", "import", addr, "-"}},
{mbox, []string{"archive", "import", addr, "-"}},
{"", []string{"member", "rm", addr, "new@x.com"}},
{"", []string{"list", "rm", addr, "--yes"}},
} {
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, c.stdin, c.args...)
if code != 0 {
t.Fatalf("%v: exit %d %s", c.args, code, errs)
}
noControl(t, strings.Join(c.args, " "), out+errs)
}
}
func TestControlCharactersInImportReportsAreEscaped(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL},
"bad"+esc+"@x\n", "member", "import", "dev@example.org", "-")
if code != 1 || !strings.Contains(errs, `line 1: bad\x1b[31mRED`) {
t.Fatalf("member import: %d %q", code, errs)
}
noControl(t, "member import", out+errs)
fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" {
b, _ := json.Marshal(map[string]any{"imported": 0, "failed": 1, "errors": []string{"message 1: " + esc}})
w.Write(b)
return
}
w.Write([]byte(`[{"id":1,"address":"dev@example.org"}]`))
}))
defer fake.Close()
var o, eb bytes.Buffer
vars := map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": fake.URL}
Run([]string{"archive", "import", "dev@example.org", "-"}, IO{In: strings.NewReader("x"), Out: &o, Err: &eb}, func(k string) string { return vars[k] })
if !strings.Contains(eb.String(), `message 1: \x1b[31mRED`) {
t.Fatalf("archive import errors: %q", eb.String())
}
noControl(t, "archive import", o.String()+eb.String())
}
func TestControlCharactersInAPIErrorsAndImportReportsAreEscaped(t *testing.T) {
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadGateway)
msg, _ := json.Marshal(map[string]string{"error": "relay said " + esc})
w.Write(msg)
}))
defer api.Close()
var out, errb bytes.Buffer
vars := map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": api.URL}
code := Run([]string{"list", "ls"}, IO{In: strings.NewReader(""), Out: &out, Err: &errb}, func(k string) string { return vars[k] })
if code != 1 || !strings.Contains(errb.String(), `relay said \x1b[31mRED`) {
t.Fatalf("exit %d: %q", code, errb.String())
}
noControl(t, "error text", errb.String())
}
func TestTokenIsNotSentOverPlainHTTPToARemoteHost(t *testing.T) {
e := newEnv(t)
run := func(url string, extra ...string) (int, string) {
args := append(append([]string{"--url", url}, extra...), "list", "ls")
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token}, "", args...)
if out != "" {
t.Fatalf("stdout %q", out)
}
return code, errs
}
code, errs := run("http://lists.example.org:8080")
if code != 2 || !strings.Contains(errs, "--insecure") || !strings.Contains(errs, "http") {
t.Fatalf("remote http must be refused with a usage error: %d %q", code, errs)
}
if code, errs = run("HTTP://10.1.2.3"); code != 2 {
t.Fatalf("scheme case must not matter: %d %q", code, errs)
}
// Past the check the only thing left to fail is the connection (port 1 is closed, .invalid never resolves).
for _, u := range []string{"http://localhost:1", "http://LocalHost.:1", "http://127.0.0.1:1", "http://[::1]:1", "https://nowhere.invalid"} {
if code, errs = run(u); code != 1 || !strings.Contains(errs, "cannot reach") {
t.Errorf("%s must be allowed: %d %q", u, code, errs)
}
}
if code, errs = run("http://nowhere.invalid", "--insecure"); code != 1 || !strings.Contains(errs, "cannot reach") {
t.Fatalf("--insecure allows plain http: %d %q", code, errs)
}
// The environment variable is checked like the flag.
code, _, errs = e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": "http://lists.example.org"}, "", "list", "ls")
if code != 2 {
t.Fatalf("PARSIMONY_URL: %d %q", code, errs)
}
// No token to protect, no URL check: offline commands never build the client.
code, _, errs = e.runEnv(t, nil, "", "--url", "http://lists.example.org", "dkim", "record", "example.org", "--key", filepath.Join(t.TempDir(), "none"))
if strings.Contains(errs, "--insecure") {
t.Fatalf("offline commands must not care about the URL")
}
}

View file

@ -135,7 +135,7 @@ func (c *cmd) listSet(args []string) int {
if c.json {
return c.printJSON(raw)
}
fmt.Fprintf(c.io.Out, "Updated %s\n", l.Address)
fmt.Fprintf(c.io.Out, "Updated %s\n", printable(l.Address))
return 0
}
@ -168,7 +168,7 @@ func (c *cmd) done(msg string) int {
if c.json {
return c.printJSON(nil)
}
fmt.Fprintln(c.io.Out, msg)
fmt.Fprintln(c.io.Out, printable(msg))
return 0
}
@ -220,7 +220,7 @@ func (c *cmd) memberLs(args []string) int {
}
if *emails {
for _, m := range ms {
fmt.Fprintln(c.io.Out, m.Address)
fmt.Fprintln(c.io.Out, printable(m.Address))
}
return 0
}
@ -259,7 +259,7 @@ func (c *cmd) memberAdd(args []string) int {
}
var m member
json.Unmarshal(raw, &m)
fmt.Fprintf(c.io.Out, "Added %s to %s\n", m.Address, l.Address)
fmt.Fprintf(c.io.Out, "Added %s to %s\n", printable(m.Address), printable(l.Address))
return 0
}
@ -327,7 +327,7 @@ func (c *cmd) memberImport(args []string) int {
}
if _, err := c.addMember(l.ID, address, name); err != nil {
failed++
fmt.Fprintf(c.io.Err, "line %d: %s: %v\n", n, line, err)
fmt.Fprintf(c.io.Err, "line %d: %s\n", n, printable(fmt.Sprintf("%s: %v", line, err)))
continue
}
imported++
@ -339,7 +339,7 @@ func (c *cmd) memberImport(args []string) int {
if failed > 0 {
msg += fmt.Sprintf(", %d failed", failed)
}
fmt.Fprintln(c.io.Out, msg)
fmt.Fprintln(c.io.Out, printable(msg))
if failed > 0 {
return 1
}
@ -544,7 +544,7 @@ func (c *cmd) archiveShow(args []string) int {
if err := json.Unmarshal(raw, &m); err != nil {
return err
}
fmt.Fprintf(c.io.Out, "Subject: %s\nFrom: %s\nDate: %s\n\n%s\n", m.Subject, m.Sender, formatDate(m.Date), m.Body)
fmt.Fprintf(c.io.Out, "Subject: %s\nFrom: %s\nDate: %s\n\n%s\n", printable(m.Subject), printable(m.Sender), formatDate(m.Date), printable(m.Body))
return nil
}, "")
}
@ -601,10 +601,10 @@ func (c *cmd) archiveImport(args []string) int {
if sum.Failed > 0 {
msg += fmt.Sprintf(", %d failed", sum.Failed)
}
fmt.Fprintln(c.io.Out, msg)
fmt.Fprintln(c.io.Out, printable(msg))
}
for _, e := range sum.Errors {
fmt.Fprintln(c.io.Err, e)
fmt.Fprintln(c.io.Err, printable(e))
}
if sum.Failed > len(sum.Errors) {
fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors))

View file

@ -67,14 +67,45 @@ func NewReader(r io.Reader, maxSize int) *Reader {
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
}
// readLine returns the next line including its newline. At the end of the input
// it returns the last partial line (if any) with io.EOF on the following call.
func (r *Reader) readLine() ([]byte, error) {
line, err := r.br.ReadBytes('\n')
if err == io.EOF && len(line) > 0 {
return line, nil
// minKeep is the least a line may be cut to, so a "From " separator is never cut short.
const minKeep = 1024
// keep is how many bytes of a line are worth holding: a message with a longer line is too
// large anyway, so the rest is read and thrown away instead of buffered.
func (r *Reader) keep() int {
if r.max <= 0 {
return 0 // no limit
}
return max(r.max+1, minKeep)
}
// readLine returns the next line including its newline, cut to keep bytes (0 means no cut),
// and the full length of the line. At the end of the input it returns the last partial line
// (if any) with io.EOF on the following call.
func (r *Reader) readLine(keep int) ([]byte, int, error) {
var line []byte
n := 0
for {
chunk, err := r.br.ReadSlice('\n')
n += len(chunk)
if room := keep - len(line); keep == 0 || room > 0 {
if keep > 0 && len(chunk) > room {
chunk = chunk[:room]
}
line = append(line, chunk...)
}
switch err {
case nil:
return line, n, nil
case bufio.ErrBufferFull:
continue
case io.EOF:
if n > 0 {
return line, n, nil
}
}
return line, n, err
}
return line, err
}
// Next returns the next message, or io.EOF when there are no more.
@ -84,7 +115,7 @@ func (r *Reader) Next() (Message, error) {
}
// Skip anything before the first separator.
for r.next == "" {
line, err := r.readLine()
line, _, err := r.readLine(r.keep())
if err != nil {
r.eof = true
if err == io.EOF {
@ -102,8 +133,9 @@ func (r *Reader) Next() (Message, error) {
var buf bytes.Buffer
oversized := false
size := 0
for {
line, err := r.readLine()
line, n, err := r.readLine(r.keep())
if err != nil {
r.eof = true
if err != io.EOF {
@ -120,12 +152,15 @@ func (r *Reader) Next() (Message, error) {
}
if escaped.Match(line) {
line = line[1:]
n--
}
buf.Write(line)
if r.max > 0 && buf.Len() > r.max {
size += n
if r.max > 0 && size > r.max {
oversized = true
buf.Reset()
continue
}
buf.Write(line)
}
if oversized {
msg.Err = ErrTooLarge

View file

@ -166,3 +166,93 @@ func TestReadErrorsArePropagated(t *testing.T) {
type errReader struct{}
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }
func TestAVeryLongLineIsTooLargeAndTheReaderResynchronises(t *testing.T) {
long := strings.Repeat("a", 5000) // far above the limit of 100, with no way to be a separator
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n\n" +
"From b@x.com Wed Jan 3 15:04:05 2024\nSubject: ok\n\nfine\n"
ms := readAll(t, in, 100)
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[0].Data != nil {
t.Fatalf("messages: %+v", ms)
}
if ms[1].Err != nil || !strings.Contains(string(ms[1].Data), "fine") {
t.Fatalf("the next message must be read: %+v", ms[1])
}
}
func TestALongLineAtTheEndOfTheInputIsTooLarge(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + strings.Repeat("a", 5000) // no newline
ms := readAll(t, in, 100)
if len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
t.Fatalf("messages: %+v", ms)
}
}
func TestAnOverlongLineBeforeTheFirstSeparatorIsSkipped(t *testing.T) {
in := strings.Repeat("x", 5000) + "\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"
ms := readAll(t, in, 100)
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), "body") {
t.Fatalf("messages: %+v", ms)
}
}
func TestLongLinesBelowTheLimitStillWork(t *testing.T) {
long := strings.Repeat("a", 3000)
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n"
ms := readAll(t, in, 4000)
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), long+"\n") {
t.Fatalf("messages: %+v", ms)
}
if ms = readAll(t, in, 0); len(ms) != 1 || ms[0].Err != nil {
t.Fatalf("no limit: %+v", ms)
}
}
func TestReadLineKeepsAtMostTheLimitButReportsTheFullLength(t *testing.T) {
// A line far longer than the reader's buffer: only keep bytes may be held.
in := strings.Repeat("a", 300_000) + "\nnext\n"
r := NewReader(strings.NewReader(in), 0)
line, n, err := r.readLine(1000)
if err != nil || n != 300_001 || len(line) > 1000 {
t.Fatalf("n=%d kept=%d err=%v", n, len(line), err)
}
if line, n, err = r.readLine(1000); err != nil || string(line) != "next\n" || n != 5 {
t.Fatalf("the following line: %q %d %v", line, n, err)
}
if _, _, err = r.readLine(1000); err != io.EOF {
t.Fatalf("want EOF, got %v", err)
}
// keep 0 means keep everything.
r = NewReader(strings.NewReader(in), 0)
if line, n, _ = r.readLine(0); len(line) != 300_001 || n != 300_001 {
t.Fatalf("unlimited: %d %d", len(line), n)
}
}
func TestASmallLimitDoesNotCutSeparatorsShort(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nlong enough to be too large\n" +
"From b@x.com Wed Jan 3 15:04:05 2024\nok\n"
ms := readAll(t, in, 10)
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[1].Err != nil || string(ms[1].Data) != "ok\n" {
t.Fatalf("messages: %+v", ms)
}
}
func TestKeepIsBoundedByTheLimit(t *testing.T) {
for _, c := range []struct{ max, want int }{{0, 0}, {-1, 0}, {100, minKeep}, {5000, 5001}} {
if got := NewReader(strings.NewReader(""), c.max).keep(); got != c.want {
t.Errorf("max %d: keep %d, want %d", c.max, got, c.want)
}
}
}
func TestTheLimitIsExactAndCountsTheRestoredLine(t *testing.T) {
body := "Subject: s\n\n>From here\n" // restored to "From here\n": 22 bytes
in := "From a@x.com Tue Jan 2 15:04:05 2024\n" + body
if ms := readAll(t, in, 22); len(ms) != 1 || ms[0].Err != nil {
t.Fatalf("exactly at the limit must pass: %+v", ms)
}
if ms := readAll(t, in, 21); len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
t.Fatalf("one over the limit must be too large: %+v", ms)
}
}

View file

@ -22,6 +22,9 @@ const (
defaultBatch = 50
defaultTimeout = time.Minute
defaultInterval = 5 * time.Second
// purgeEvery is the least time between two scans that forget old failed deliveries.
purgeEvery = 5 * time.Minute
)
var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour}
@ -61,6 +64,53 @@ type Worker struct {
wakeOnce sync.Once
wake chan struct{}
runMu sync.Mutex
lastPurge time.Time // under runMu; zero until the first purge
// Hooks for tests; nil means the Store methods.
purge func() (int, error)
readBody func(msgID int64) ([]byte, error)
}
// bodies reads each message body once per run, however many recipients share it.
type bodies struct {
read func(msgID int64) ([]byte, error)
mu sync.Mutex
m map[int64]*body
}
type body struct {
once sync.Once
data []byte
err error
}
func (b *bodies) get(msgID int64) ([]byte, error) {
b.mu.Lock()
e := b.m[msgID]
if e == nil {
e = &body{}
b.m[msgID] = e
}
b.mu.Unlock()
e.once.Do(func() { e.data, e.err = b.read(msgID) })
return e.data, e.err
}
// purgeOld forgets old failed deliveries, at most once every purgeEvery.
func (w *Worker) purgeOld() {
now := w.Store.Now()
if !w.lastPurge.IsZero() && now.Sub(w.lastPurge) < purgeEvery {
return
}
purge := w.purge
if purge == nil {
purge = w.Store.PurgeFailedOutbox
}
if _, err := purge(); err != nil {
log.Printf("queue: purge old failures: %v", err)
return // try again on the next run
}
w.lastPurge = now
}
func (w *Worker) wakeCh() chan struct{} {
@ -125,14 +175,18 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
if conc <= 0 {
conc = defaultConcurrency
}
if _, err := w.Store.PurgeFailedOutbox(); err != nil {
log.Printf("queue: purge old failures: %v", err)
}
w.purgeOld()
due, err := w.Store.DueOutbox(batch)
if err != nil {
return Stats{}, err
}
read := w.readBody
if read == nil {
read = w.Store.OutboxData
}
cache := &bodies{read: read, m: map[int64]*body{}}
var (
mu sync.Mutex
stats Stats
@ -148,7 +202,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
go func(it store.OutboxItem) {
defer wg.Done()
defer func() { <-sem }()
o := w.deliver(ctx, it)
o := w.deliver(ctx, cache, it)
mu.Lock()
stats.Sent += o.Sent
stats.Retried += o.Retried
@ -162,8 +216,8 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
return stats, nil
}
func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
data, err := w.Store.OutboxData(it.MsgID)
func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats {
data, err := cache.get(it.MsgID)
if errors.Is(err, store.ErrNotFound) {
w.complete(it)
return Stats{Dropped: 1}

View file

@ -558,3 +558,113 @@ func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) {
t.Fatalf("every delivery stays listed as failed: %+v", stats)
}
}
func TestPurgeRunsOncePerIntervalNotOnEveryBatch(t *testing.T) {
e := setup(t)
var purges int32
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, nil }
e.w.Batch = 1
e.enqueue(t, 3)
for i := 0; i < 3; i++ {
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 1 {
t.Fatalf("run %d: %+v %v", i, st, err)
}
}
if n := atomic.LoadInt32(&purges); n != 1 {
t.Fatalf("purge ran %d times in 3 batches, want 1", n)
}
e.clk.set(t0.Add(purgeEvery - time.Second))
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 1 {
t.Fatalf("purge ran again before the interval: %d", n)
}
e.clk.set(t0.Add(purgeEvery))
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("purge must run again after the interval: %d", n)
}
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("interval counts from the last purge: %d", n)
}
}
func TestFailedPurgeIsRetriedOnTheNextRun(t *testing.T) {
e := setup(t)
var purges int32
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, errors.New("db busy") }
e.w.RunOnce(context.Background())
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("a failed purge must not delay the next attempt: %d runs", n)
}
}
func TestOneBodyReadServesEveryRecipientOfAMessage(t *testing.T) {
e := setup(t)
reads := map[int64]int{}
var mu sync.Mutex
e.w.readBody = func(id int64) ([]byte, error) {
mu.Lock()
reads[id]++
mu.Unlock()
return e.s.OutboxData(id)
}
e.enqueue(t, 6)
e.w.Concurrency = 6
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 6 {
t.Fatalf("run: %+v %v", st, err)
}
if len(reads) != 1 {
t.Fatalf("reads = %v, want one message", reads)
}
for id, n := range reads {
if n != 1 {
t.Fatalf("message %d read %d times, want 1", id, n)
}
}
for _, m := range e.snd.sent {
if !strings.HasSuffix(string(m.Data), "Subject: hi\r\n\r\nbody\r\n") {
t.Fatalf("data = %q", m.Data)
}
}
// The cache lives for one run only: a later run reads again.
e.enqueue(t, 1)
e.w.RunOnce(context.Background())
total := 0
for _, n := range reads {
total += n
}
if total != 2 {
t.Fatalf("the next run must read its own body: %v", reads)
}
}
func TestEachDistinctMessageIsReadOnce(t *testing.T) {
e := setup(t)
var calls int32
e.w.readBody = func(id int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return e.s.OutboxData(id) }
for i := 0; i < 3; i++ {
e.enqueue(t, 2)
}
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 6 {
t.Fatalf("sent %d", st.Sent)
}
if n := atomic.LoadInt32(&calls); n != 3 {
t.Fatalf("%d reads for 3 messages", n)
}
}
func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) {
e := setup(t)
var calls int32
e.w.readBody = func(int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return nil, store.ErrNotFound }
e.enqueue(t, 3)
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 3 || e.snd.count() != 0 {
t.Fatalf("stats %+v", st)
}
if n := atomic.LoadInt32(&calls); n != 1 {
t.Fatalf("%d reads", n)
}
}

View file

@ -7,6 +7,7 @@ import (
"io"
"mime"
"net/mail"
"strings"
"github.com/emersion/go-message/textproto"
@ -23,9 +24,17 @@ func rewrite(l store.List, data []byte) ([]byte, error) {
}
// Never trust list headers coming from outside.
for _, k := range []string{"List-Id", "List-Post", "List-Unsubscribe", "List-Unsubscribe-Post", "Precedence"} {
for _, k := range []string{"List-Id", "List-Post", "List-Unsubscribe", "List-Unsubscribe-Post", "Precedence",
"List-Help", "List-Archive", "List-Owner", "List-Subscribe",
// The author's signature and authentication results do not survive our changes: they only confuse receivers.
"DKIM-Signature", "Authentication-Results"} {
h.Del(k)
}
for f := h.Fields(); f.Next(); {
if strings.HasPrefix(strings.ToLower(f.Key()), "arc-") {
f.Del()
}
}
id := l.Name + "." + l.Domain
// The phrase is RFC 2047 encoded when it is not ASCII; the <id> stays plain ASCII (list names and domains are).

View file

@ -585,3 +585,46 @@ func TestLeaveConfirmationWorksOnAClosedList(t *testing.T) {
t.Fatal("alice must be gone")
}
}
func TestIncomingListManagementAndAuthenticationHeadersAreStripped(t *testing.T) {
r := setup(t)
msg := "From: Alice <alice@example.com>\r\n" +
"List-Help: <mailto:evil@x.com?subject=help>\r\nList-Archive: <https://evil.example/a>\r\n" +
"List-Owner: <mailto:evil@x.com>\r\nList-Subscribe: <https://evil.example/s>\r\n" +
"DKIM-Signature: v=1; a=rsa-sha256; d=example.com; s=x; h=from; bh=aa; b=bb\r\n" +
"Authentication-Results: mx.example.com; dkim=pass\r\n" +
"ARC-Seal: i=1; a=rsa-sha256; cv=none; d=example.com; s=x; b=cc\r\n" +
"ARC-Message-Signature: i=1; a=rsa-sha256; d=example.com; s=x; b=dd\r\n" +
"ARC-Authentication-Results: i=1; mx.example.com; dkim=pass\r\n" +
"Subject: hi\r\n\r\nx\r\n"
res, err := r.Route("alice@example.com", "dev@example.org", []byte(msg))
if err != nil {
t.Fatal(err)
}
h := header(t, res.Data)
for _, k := range []string{"List-Help", "List-Archive", "List-Owner", "List-Subscribe", "DKIM-Signature",
"Authentication-Results", "ARC-Seal", "ARC-Message-Signature", "ARC-Authentication-Results"} {
if v := h.Get(k); v != "" { // Get canonicalizes the key
t.Errorf("%s survived: %v", k, v)
}
}
if h.Get("Subject") != "hi" {
t.Errorf("unrelated headers must stay: %v", h)
}
}
func TestDroppingARCHeadersKeepsSimilarlyNamedOnes(t *testing.T) {
r := setup(t)
msg := "From: Alice <alice@example.com>\r\nARC-Seal: i=1\r\nArcade: keep\r\nX-ARC-Note: keep\r\nARC-Seal: i=2\r\n\r\nx\r\n"
res, err := r.Route("alice@example.com", "dev@example.org", []byte(msg))
if err != nil {
t.Fatal(err)
}
h := header(t, res.Data)
if v := h["Arc-Seal"]; len(v) != 0 {
t.Errorf("every ARC-Seal must go: %v", v)
}
if h.Get("Arcade") != "keep" || h.Get("X-Arc-Note") != "keep" {
t.Errorf("only the ARC- prefix is dropped: %v", h)
}
}