diff --git a/README.md b/README.md index 0fe89e9..d635e9a 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ parsimony help `` is an address or a numeric id. Destructive commands (`list rm`, `archive rm`) need `--yes`. Exit status: 0 success, 1 failure (including lines that failed in `member import`), 2 usage error. +The client refuses to send the token over plain `http://` to any host other than `localhost`, `127.0.0.1` or `::1` +(usage error, exit 2): put the API behind HTTPS, or pass `--insecure` if you really mean it. +Subjects, senders, message bodies and error texts that come from mail are printed with control characters +(escape sequences) shown as `\x1b`-style text instead of being sent to the terminal; `--json` output is already escaped. `parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works. In the container: `docker exec parsimony /parsimony list ls`. diff --git a/admin/src/tabs/HeldTab.test.tsx b/admin/src/tabs/HeldTab.test.tsx index 5abdc84..e1e8c3b 100644 --- a/admin/src/tabs/HeldTab.test.tsx +++ b/admin/src/tabs/HeldTab.test.tsx @@ -76,15 +76,21 @@ describe('held tab', () => { expect(await screen.findByText(/nothing is waiting/i)).toBeInTheDocument() }) - it('explains how messages get held when posting is not moderated', async () => { - await setup([], aList({ id: 1, post_policy: 'members' })) - expect(await screen.findByText(/set posting to moderated/i)).toBeInTheDocument() + it('explains both ways a message gets held, whatever the posting policy', async () => { + for (const post_policy of ['members', 'moderated'] as const) { + const { unmount } = await setup([], aList({ id: 1, post_policy })) + const hint = await screen.findByText(/messages are held when/i) + expect(hint).toHaveTextContent(/posting is set to moderated/i) + expect(hint).toHaveTextContent(/from header does not match the sender/i) + expect(hint).not.toHaveTextContent(/only/i) + unmount() + } }) - it('does not nag about the policy when the list is moderated', async () => { - await setup([]) - await screen.findByText(/nothing is waiting/i) - expect(screen.queryByText(/set posting to moderated/i)).not.toBeInTheDocument() + it('shows no hint when there are held messages', async () => { + await setup([held(1)]) + await screen.findByText('Subject 1') + expect(screen.queryByText(/messages are held when/i)).not.toBeInTheDocument() }) it('approves a message: it is sent and leaves the queue', async () => { diff --git a/admin/src/tabs/HeldTab.tsx b/admin/src/tabs/HeldTab.tsx index fa3c0e6..4b43188 100644 --- a/admin/src/tabs/HeldTab.tsx +++ b/admin/src/tabs/HeldTab.tsx @@ -51,9 +51,10 @@ export function HeldTab() { {held.data && items.length === 0 && (

Nothing is waiting for approval.

- {list.post_policy !== 'moderated' && ( -

Messages are held only when you set Posting to moderated in Settings.

- )} +

+ Messages are held when Posting is set to moderated in Settings, or when the From header does not match + the sender. +

)} {items.length > 0 && ( diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 81e4c80..2d50292 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "io" + "net" "net/http" "net/url" "strconv" @@ -22,7 +23,7 @@ type IO struct { Out, Err io.Writer } -const usageText = `Usage: parsimony [--url URL] [--json] [arguments] +const usageText = `Usage: parsimony [--url URL] [--insecure] [--json] [arguments] Administration (talks to the REST API): list ls show all lists @@ -69,6 +70,7 @@ Environment: PARSIMONY_URL API address, default http://127.0.0.1:8080 (--url overrides) --json prints the API's JSON instead of tables. +--insecure allows sending the token over plain http to a host other than localhost. Exit status: 0 success, 1 failure, 2 usage error. ` @@ -79,6 +81,7 @@ func Run(args []string, io IO, getenv func(string) string) int { fs := newFlagSet("parsimony") fs.StringVar(&c.url, "url", "", "") fs.BoolVar(&c.json, "json", false, "") + fs.BoolVar(&c.insecure, "insecure", false, "") if err := fs.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { fmt.Fprint(io.Out, usageText) @@ -88,7 +91,7 @@ func Run(args []string, io IO, getenv func(string) string) int { } rest := fs.Args() if len(rest) == 0 { - fmt.Fprint(io.Err, "usage: parsimony [--url URL] [--json] (see parsimony help)\n") + fmt.Fprint(io.Err, "usage: parsimony [--url URL] [--insecure] [--json] (see parsimony help)\n") return 2 } if rest[0] == "help" { @@ -124,16 +127,27 @@ type cmd struct { getenv func(string) string url string json bool - cl *client + // insecure allows the token to travel over plain http to a non-loopback host. + insecure bool + cl *client } +// usageErr is an error that is the caller's mistake: exit status 2. +type usageErr string + +func (e usageErr) Error() string { return string(e) } + func (c *cmd) usageError(format string, a ...any) int { fmt.Fprintf(c.io.Err, "error: "+format+"\n", a...) return 2 } func (c *cmd) fail(err error) int { - fmt.Fprintf(c.io.Err, "error: %v\n", err) + fmt.Fprintf(c.io.Err, "error: %s\n", printable(err.Error())) + var u usageErr + if errors.As(err, &u) { + return 2 + } return 1 } @@ -153,9 +167,14 @@ func (c *cmd) client() (*client, error) { if base == "" { base = "http://127.0.0.1:8080" } - if _, err := url.Parse(base); err != nil { + u, err := url.Parse(base) + if err != nil { return nil, fmt.Errorf("invalid URL %q", base) } + // url.Parse lowercases the scheme. + if u.Scheme == "http" && !isLoopback(u.Hostname()) && !c.insecure { + return nil, usageErr(fmt.Sprintf("refusing to send the token over plain http to %s: use https, or pass --insecure", u.Host)) + } c.cl = &client{ base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 30 * time.Second}, @@ -312,10 +331,44 @@ func (c *cmd) printJSON(raw json.RawMessage) int { return 0 } +// isLoopback reports whether host is this machine: localhost or a loopback address. +func isLoopback(host string) bool { + if strings.EqualFold(strings.TrimSuffix(host, "."), "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// printable makes text that came from mail or from the API safe for a terminal: control +// characters (C0, DEL, C1) other than tab and newline are shown as \xNN / \uNNNN instead of +// being sent, so an escape sequence in a subject cannot drive the terminal. CRLF counts as a newline. +func printable(s string) string { + s = strings.ReplaceAll(s, "\r\n", "\n") + var b strings.Builder + for _, r := range s { + switch { + case r == '\t' || r == '\n': + b.WriteRune(r) + case r < 0x20 || r == 0x7f: + fmt.Fprintf(&b, `\x%02x`, r) + case r >= 0x80 && r <= 0x9f: + fmt.Fprintf(&b, `\u%04x`, r) + default: + b.WriteRune(r) + } + } + return b.String() +} + func (c *cmd) table(rows [][]string) { w := tabwriter.NewWriter(c.io.Out, 0, 8, 2, ' ', 0) for _, r := range rows { - fmt.Fprintln(w, strings.Join(r, "\t")) + cells := make([]string, len(r)) + for i, cell := range r { + cells[i] = printable(cell) + } + fmt.Fprintln(w, strings.Join(cells, "\t")) } w.Flush() } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index d205820..423f29f 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -824,3 +824,172 @@ func TestHeldListingWarnsWhenTheHeaderDiffersFromTheSender(t *testing.T) { t.Fatalf("no usable From is flagged too: %q", lines["no from"]) } } + +const esc = "\x1b[31mRED\x1b]0;pwned\x07" + +// noControl fails when s still holds a control character other than tab and newline. +func noControl(t *testing.T, what, s string) { + t.Helper() + for _, r := range s { + if (r < 0x20 && r != '\t' && r != '\n') || (r >= 0x7f && r <= 0x9f) { + t.Fatalf("%s prints control character %U:\n%q", what, r, s) + } + } +} + +func TestPrintableStripsControlCharactersButKeepsTabAndNewline(t *testing.T) { + for in, want := range map[string]string{ + "plain text, accents: àèì 日本": "plain text, accents: àèì 日本", + "a\tb\nc": "a\tb\nc", + "a\r\nb": "a\nb", + "\x1b[31mred\x1b[0m": `\x1b[31mred\x1b[0m`, + "us\x1f": `us\x1f`, + "bell\x07 nul\x00 del\x7f": `bell\x07 nul\x00 del\x7f`, + "c1 \u009b31m and \u0085": `c1 \u009b31m and \u0085`, + "c1 edges \u0080 \u009f, kept: ~ \u00a0 \u00a1": `c1 edges \u0080 \u009f, kept: ~ ` + "\u00a0 \u00a1", + "lone\rcarriage": `lone\x0dcarriage`, + "": "", + } { + if got := printable(in); got != want { + t.Errorf("printable(%q) = %q, want %q", in, got, want) + } + } +} + +func TestControlCharactersFromMailNeverReachTheTerminal(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + l, _ := e.s.FindList("dev@example.org") + e.s.ArchiveMessage(l.ID, store.Message{MessageID: "", Subject: "subj " + esc, Sender: "from " + esc, + Date: time.Unix(1_700_000_000, 0), Body: "line one\r\nbody " + esc + "\r\nlast"}) + e.s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob "+esc+" \r\nSubject: held "+esc+"\r\n\r\nb"), "reason "+esc) + e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "lost@x.com"}}) + due, _ := e.s.DueOutbox(10) + e.s.FailOutbox(due[0].ID, "refused: 554 "+esc) + + for _, args := range [][]string{ + {"archive", "ls", "dev@example.org"}, {"archive", "ls", "dev@example.org", "--q", "subj"}, + {"archive", "show", "1"}, {"held", "ls", "dev@example.org"}, {"queue", "ls"}, + } { + out := e.ok(t, args...) + noControl(t, strings.Join(args, " "), out) + if !strings.Contains(out, `\x1b[31mRED`) { + t.Errorf("%v: the escape sequence should stay visible as text:\n%s", args, out) + } + } + if out := e.ok(t, "archive", "show", "1"); !strings.Contains(out, "\nbody ") || strings.Contains(out, `\x0d`) { + t.Errorf("CRLF line ends are not control noise: %q", out) + } +} + +func TestControlCharactersInListAndMemberNamesAreEscaped(t *testing.T) { + e := newEnv(t) + addr := "a\x1b[31mb\x07@x.org" + l, err := e.s.CreateList("a\x1b[31mb\x07", "x.org", "title "+esc) + if err != nil { + t.Fatal(err) + } + e.s.AddMember(l.ID, "m\x1b[2Jm@x.org", "name "+esc) + mbox := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n" + for _, c := range []struct { + stdin string + args []string + }{ + {"", []string{"list", "ls"}}, {"", []string{"list", "show", addr}}, + {"", []string{"list", "set", addr, "--title", "t"}}, + {"", []string{"member", "ls", addr}}, {"", []string{"member", "ls", addr, "--emails"}}, + {"", []string{"member", "add", addr, "new@x.com"}}, + {"new2@x.com\n", []string{"member", "import", addr, "-"}}, + {mbox, []string{"archive", "import", addr, "-"}}, + {"", []string{"member", "rm", addr, "new@x.com"}}, + {"", []string{"list", "rm", addr, "--yes"}}, + } { + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, c.stdin, c.args...) + if code != 0 { + t.Fatalf("%v: exit %d %s", c.args, code, errs) + } + noControl(t, strings.Join(c.args, " "), out+errs) + } +} + +func TestControlCharactersInImportReportsAreEscaped(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, + "bad"+esc+"@x\n", "member", "import", "dev@example.org", "-") + if code != 1 || !strings.Contains(errs, `line 1: bad\x1b[31mRED`) { + t.Fatalf("member import: %d %q", code, errs) + } + noControl(t, "member import", out+errs) + + fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" { + b, _ := json.Marshal(map[string]any{"imported": 0, "failed": 1, "errors": []string{"message 1: " + esc}}) + w.Write(b) + return + } + w.Write([]byte(`[{"id":1,"address":"dev@example.org"}]`)) + })) + defer fake.Close() + var o, eb bytes.Buffer + vars := map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": fake.URL} + Run([]string{"archive", "import", "dev@example.org", "-"}, IO{In: strings.NewReader("x"), Out: &o, Err: &eb}, func(k string) string { return vars[k] }) + if !strings.Contains(eb.String(), `message 1: \x1b[31mRED`) { + t.Fatalf("archive import errors: %q", eb.String()) + } + noControl(t, "archive import", o.String()+eb.String()) +} + +func TestControlCharactersInAPIErrorsAndImportReportsAreEscaped(t *testing.T) { + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + msg, _ := json.Marshal(map[string]string{"error": "relay said " + esc}) + w.Write(msg) + })) + defer api.Close() + var out, errb bytes.Buffer + vars := map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": api.URL} + code := Run([]string{"list", "ls"}, IO{In: strings.NewReader(""), Out: &out, Err: &errb}, func(k string) string { return vars[k] }) + if code != 1 || !strings.Contains(errb.String(), `relay said \x1b[31mRED`) { + t.Fatalf("exit %d: %q", code, errb.String()) + } + noControl(t, "error text", errb.String()) +} + +func TestTokenIsNotSentOverPlainHTTPToARemoteHost(t *testing.T) { + e := newEnv(t) + run := func(url string, extra ...string) (int, string) { + args := append(append([]string{"--url", url}, extra...), "list", "ls") + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token}, "", args...) + if out != "" { + t.Fatalf("stdout %q", out) + } + return code, errs + } + code, errs := run("http://lists.example.org:8080") + if code != 2 || !strings.Contains(errs, "--insecure") || !strings.Contains(errs, "http") { + t.Fatalf("remote http must be refused with a usage error: %d %q", code, errs) + } + if code, errs = run("HTTP://10.1.2.3"); code != 2 { + t.Fatalf("scheme case must not matter: %d %q", code, errs) + } + // Past the check the only thing left to fail is the connection (port 1 is closed, .invalid never resolves). + for _, u := range []string{"http://localhost:1", "http://LocalHost.:1", "http://127.0.0.1:1", "http://[::1]:1", "https://nowhere.invalid"} { + if code, errs = run(u); code != 1 || !strings.Contains(errs, "cannot reach") { + t.Errorf("%s must be allowed: %d %q", u, code, errs) + } + } + if code, errs = run("http://nowhere.invalid", "--insecure"); code != 1 || !strings.Contains(errs, "cannot reach") { + t.Fatalf("--insecure allows plain http: %d %q", code, errs) + } + // The environment variable is checked like the flag. + code, _, errs = e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": "http://lists.example.org"}, "", "list", "ls") + if code != 2 { + t.Fatalf("PARSIMONY_URL: %d %q", code, errs) + } + // No token to protect, no URL check: offline commands never build the client. + code, _, errs = e.runEnv(t, nil, "", "--url", "http://lists.example.org", "dkim", "record", "example.org", "--key", filepath.Join(t.TempDir(), "none")) + if strings.Contains(errs, "--insecure") { + t.Fatalf("offline commands must not care about the URL") + } +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 0a09494..1232346 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -135,7 +135,7 @@ func (c *cmd) listSet(args []string) int { if c.json { return c.printJSON(raw) } - fmt.Fprintf(c.io.Out, "Updated %s\n", l.Address) + fmt.Fprintf(c.io.Out, "Updated %s\n", printable(l.Address)) return 0 } @@ -168,7 +168,7 @@ func (c *cmd) done(msg string) int { if c.json { return c.printJSON(nil) } - fmt.Fprintln(c.io.Out, msg) + fmt.Fprintln(c.io.Out, printable(msg)) return 0 } @@ -220,7 +220,7 @@ func (c *cmd) memberLs(args []string) int { } if *emails { for _, m := range ms { - fmt.Fprintln(c.io.Out, m.Address) + fmt.Fprintln(c.io.Out, printable(m.Address)) } return 0 } @@ -259,7 +259,7 @@ func (c *cmd) memberAdd(args []string) int { } var m member json.Unmarshal(raw, &m) - fmt.Fprintf(c.io.Out, "Added %s to %s\n", m.Address, l.Address) + fmt.Fprintf(c.io.Out, "Added %s to %s\n", printable(m.Address), printable(l.Address)) return 0 } @@ -327,7 +327,7 @@ func (c *cmd) memberImport(args []string) int { } if _, err := c.addMember(l.ID, address, name); err != nil { failed++ - fmt.Fprintf(c.io.Err, "line %d: %s: %v\n", n, line, err) + fmt.Fprintf(c.io.Err, "line %d: %s\n", n, printable(fmt.Sprintf("%s: %v", line, err))) continue } imported++ @@ -339,7 +339,7 @@ func (c *cmd) memberImport(args []string) int { if failed > 0 { msg += fmt.Sprintf(", %d failed", failed) } - fmt.Fprintln(c.io.Out, msg) + fmt.Fprintln(c.io.Out, printable(msg)) if failed > 0 { return 1 } @@ -544,7 +544,7 @@ func (c *cmd) archiveShow(args []string) int { if err := json.Unmarshal(raw, &m); err != nil { return err } - fmt.Fprintf(c.io.Out, "Subject: %s\nFrom: %s\nDate: %s\n\n%s\n", m.Subject, m.Sender, formatDate(m.Date), m.Body) + fmt.Fprintf(c.io.Out, "Subject: %s\nFrom: %s\nDate: %s\n\n%s\n", printable(m.Subject), printable(m.Sender), formatDate(m.Date), printable(m.Body)) return nil }, "") } @@ -601,10 +601,10 @@ func (c *cmd) archiveImport(args []string) int { if sum.Failed > 0 { msg += fmt.Sprintf(", %d failed", sum.Failed) } - fmt.Fprintln(c.io.Out, msg) + fmt.Fprintln(c.io.Out, printable(msg)) } for _, e := range sum.Errors { - fmt.Fprintln(c.io.Err, e) + fmt.Fprintln(c.io.Err, printable(e)) } if sum.Failed > len(sum.Errors) { fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors)) diff --git a/internal/mbox/mbox.go b/internal/mbox/mbox.go index a6aa4b6..1f86df9 100644 --- a/internal/mbox/mbox.go +++ b/internal/mbox/mbox.go @@ -67,14 +67,45 @@ func NewReader(r io.Reader, maxSize int) *Reader { return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize} } -// readLine returns the next line including its newline. At the end of the input -// it returns the last partial line (if any) with io.EOF on the following call. -func (r *Reader) readLine() ([]byte, error) { - line, err := r.br.ReadBytes('\n') - if err == io.EOF && len(line) > 0 { - return line, nil +// minKeep is the least a line may be cut to, so a "From " separator is never cut short. +const minKeep = 1024 + +// keep is how many bytes of a line are worth holding: a message with a longer line is too +// large anyway, so the rest is read and thrown away instead of buffered. +func (r *Reader) keep() int { + if r.max <= 0 { + return 0 // no limit + } + return max(r.max+1, minKeep) +} + +// readLine returns the next line including its newline, cut to keep bytes (0 means no cut), +// and the full length of the line. At the end of the input it returns the last partial line +// (if any) with io.EOF on the following call. +func (r *Reader) readLine(keep int) ([]byte, int, error) { + var line []byte + n := 0 + for { + chunk, err := r.br.ReadSlice('\n') + n += len(chunk) + if room := keep - len(line); keep == 0 || room > 0 { + if keep > 0 && len(chunk) > room { + chunk = chunk[:room] + } + line = append(line, chunk...) + } + switch err { + case nil: + return line, n, nil + case bufio.ErrBufferFull: + continue + case io.EOF: + if n > 0 { + return line, n, nil + } + } + return line, n, err } - return line, err } // Next returns the next message, or io.EOF when there are no more. @@ -84,7 +115,7 @@ func (r *Reader) Next() (Message, error) { } // Skip anything before the first separator. for r.next == "" { - line, err := r.readLine() + line, _, err := r.readLine(r.keep()) if err != nil { r.eof = true if err == io.EOF { @@ -102,8 +133,9 @@ func (r *Reader) Next() (Message, error) { var buf bytes.Buffer oversized := false + size := 0 for { - line, err := r.readLine() + line, n, err := r.readLine(r.keep()) if err != nil { r.eof = true if err != io.EOF { @@ -120,12 +152,15 @@ func (r *Reader) Next() (Message, error) { } if escaped.Match(line) { line = line[1:] + n-- } - buf.Write(line) - if r.max > 0 && buf.Len() > r.max { + size += n + if r.max > 0 && size > r.max { oversized = true buf.Reset() + continue } + buf.Write(line) } if oversized { msg.Err = ErrTooLarge diff --git a/internal/mbox/mbox_test.go b/internal/mbox/mbox_test.go index a39e7e5..365d242 100644 --- a/internal/mbox/mbox_test.go +++ b/internal/mbox/mbox_test.go @@ -166,3 +166,93 @@ func TestReadErrorsArePropagated(t *testing.T) { type errReader struct{} func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") } + +func TestAVeryLongLineIsTooLargeAndTheReaderResynchronises(t *testing.T) { + long := strings.Repeat("a", 5000) // far above the limit of 100, with no way to be a separator + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n\n" + + "From b@x.com Wed Jan 3 15:04:05 2024\nSubject: ok\n\nfine\n" + ms := readAll(t, in, 100) + if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[0].Data != nil { + t.Fatalf("messages: %+v", ms) + } + if ms[1].Err != nil || !strings.Contains(string(ms[1].Data), "fine") { + t.Fatalf("the next message must be read: %+v", ms[1]) + } +} + +func TestALongLineAtTheEndOfTheInputIsTooLarge(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + strings.Repeat("a", 5000) // no newline + ms := readAll(t, in, 100) + if len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) { + t.Fatalf("messages: %+v", ms) + } +} + +func TestAnOverlongLineBeforeTheFirstSeparatorIsSkipped(t *testing.T) { + in := strings.Repeat("x", 5000) + "\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n" + ms := readAll(t, in, 100) + if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), "body") { + t.Fatalf("messages: %+v", ms) + } +} + +func TestLongLinesBelowTheLimitStillWork(t *testing.T) { + long := strings.Repeat("a", 3000) + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n" + ms := readAll(t, in, 4000) + if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), long+"\n") { + t.Fatalf("messages: %+v", ms) + } + if ms = readAll(t, in, 0); len(ms) != 1 || ms[0].Err != nil { + t.Fatalf("no limit: %+v", ms) + } +} + +func TestReadLineKeepsAtMostTheLimitButReportsTheFullLength(t *testing.T) { + // A line far longer than the reader's buffer: only keep bytes may be held. + in := strings.Repeat("a", 300_000) + "\nnext\n" + r := NewReader(strings.NewReader(in), 0) + line, n, err := r.readLine(1000) + if err != nil || n != 300_001 || len(line) > 1000 { + t.Fatalf("n=%d kept=%d err=%v", n, len(line), err) + } + if line, n, err = r.readLine(1000); err != nil || string(line) != "next\n" || n != 5 { + t.Fatalf("the following line: %q %d %v", line, n, err) + } + if _, _, err = r.readLine(1000); err != io.EOF { + t.Fatalf("want EOF, got %v", err) + } + // keep 0 means keep everything. + r = NewReader(strings.NewReader(in), 0) + if line, n, _ = r.readLine(0); len(line) != 300_001 || n != 300_001 { + t.Fatalf("unlimited: %d %d", len(line), n) + } +} + +func TestASmallLimitDoesNotCutSeparatorsShort(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nlong enough to be too large\n" + + "From b@x.com Wed Jan 3 15:04:05 2024\nok\n" + ms := readAll(t, in, 10) + if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[1].Err != nil || string(ms[1].Data) != "ok\n" { + t.Fatalf("messages: %+v", ms) + } +} + +func TestKeepIsBoundedByTheLimit(t *testing.T) { + for _, c := range []struct{ max, want int }{{0, 0}, {-1, 0}, {100, minKeep}, {5000, 5001}} { + if got := NewReader(strings.NewReader(""), c.max).keep(); got != c.want { + t.Errorf("max %d: keep %d, want %d", c.max, got, c.want) + } + } +} + +func TestTheLimitIsExactAndCountsTheRestoredLine(t *testing.T) { + body := "Subject: s\n\n>From here\n" // restored to "From here\n": 22 bytes + in := "From a@x.com Tue Jan 2 15:04:05 2024\n" + body + if ms := readAll(t, in, 22); len(ms) != 1 || ms[0].Err != nil { + t.Fatalf("exactly at the limit must pass: %+v", ms) + } + if ms := readAll(t, in, 21); len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) { + t.Fatalf("one over the limit must be too large: %+v", ms) + } +} diff --git a/internal/queue/queue.go b/internal/queue/queue.go index 201ddf2..9976a9c 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -22,6 +22,9 @@ const ( defaultBatch = 50 defaultTimeout = time.Minute defaultInterval = 5 * time.Second + + // purgeEvery is the least time between two scans that forget old failed deliveries. + purgeEvery = 5 * time.Minute ) var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour} @@ -61,6 +64,53 @@ type Worker struct { wakeOnce sync.Once wake chan struct{} runMu sync.Mutex + + lastPurge time.Time // under runMu; zero until the first purge + // Hooks for tests; nil means the Store methods. + purge func() (int, error) + readBody func(msgID int64) ([]byte, error) +} + +// bodies reads each message body once per run, however many recipients share it. +type bodies struct { + read func(msgID int64) ([]byte, error) + mu sync.Mutex + m map[int64]*body +} + +type body struct { + once sync.Once + data []byte + err error +} + +func (b *bodies) get(msgID int64) ([]byte, error) { + b.mu.Lock() + e := b.m[msgID] + if e == nil { + e = &body{} + b.m[msgID] = e + } + b.mu.Unlock() + e.once.Do(func() { e.data, e.err = b.read(msgID) }) + return e.data, e.err +} + +// purgeOld forgets old failed deliveries, at most once every purgeEvery. +func (w *Worker) purgeOld() { + now := w.Store.Now() + if !w.lastPurge.IsZero() && now.Sub(w.lastPurge) < purgeEvery { + return + } + purge := w.purge + if purge == nil { + purge = w.Store.PurgeFailedOutbox + } + if _, err := purge(); err != nil { + log.Printf("queue: purge old failures: %v", err) + return // try again on the next run + } + w.lastPurge = now } func (w *Worker) wakeCh() chan struct{} { @@ -125,14 +175,18 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { if conc <= 0 { conc = defaultConcurrency } - if _, err := w.Store.PurgeFailedOutbox(); err != nil { - log.Printf("queue: purge old failures: %v", err) - } + w.purgeOld() due, err := w.Store.DueOutbox(batch) if err != nil { return Stats{}, err } + read := w.readBody + if read == nil { + read = w.Store.OutboxData + } + cache := &bodies{read: read, m: map[int64]*body{}} + var ( mu sync.Mutex stats Stats @@ -148,7 +202,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { go func(it store.OutboxItem) { defer wg.Done() defer func() { <-sem }() - o := w.deliver(ctx, it) + o := w.deliver(ctx, cache, it) mu.Lock() stats.Sent += o.Sent stats.Retried += o.Retried @@ -162,8 +216,8 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { return stats, nil } -func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats { - data, err := w.Store.OutboxData(it.MsgID) +func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats { + data, err := cache.get(it.MsgID) if errors.Is(err, store.ErrNotFound) { w.complete(it) return Stats{Dropped: 1} diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 7cd9f76..7e2dbd2 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -558,3 +558,113 @@ func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) { t.Fatalf("every delivery stays listed as failed: %+v", stats) } } + +func TestPurgeRunsOncePerIntervalNotOnEveryBatch(t *testing.T) { + e := setup(t) + var purges int32 + e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, nil } + e.w.Batch = 1 + e.enqueue(t, 3) + for i := 0; i < 3; i++ { + if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 1 { + t.Fatalf("run %d: %+v %v", i, st, err) + } + } + if n := atomic.LoadInt32(&purges); n != 1 { + t.Fatalf("purge ran %d times in 3 batches, want 1", n) + } + e.clk.set(t0.Add(purgeEvery - time.Second)) + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 1 { + t.Fatalf("purge ran again before the interval: %d", n) + } + e.clk.set(t0.Add(purgeEvery)) + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 2 { + t.Fatalf("purge must run again after the interval: %d", n) + } + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 2 { + t.Fatalf("interval counts from the last purge: %d", n) + } +} + +func TestFailedPurgeIsRetriedOnTheNextRun(t *testing.T) { + e := setup(t) + var purges int32 + e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, errors.New("db busy") } + e.w.RunOnce(context.Background()) + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 2 { + t.Fatalf("a failed purge must not delay the next attempt: %d runs", n) + } +} + +func TestOneBodyReadServesEveryRecipientOfAMessage(t *testing.T) { + e := setup(t) + reads := map[int64]int{} + var mu sync.Mutex + e.w.readBody = func(id int64) ([]byte, error) { + mu.Lock() + reads[id]++ + mu.Unlock() + return e.s.OutboxData(id) + } + e.enqueue(t, 6) + e.w.Concurrency = 6 + if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 6 { + t.Fatalf("run: %+v %v", st, err) + } + if len(reads) != 1 { + t.Fatalf("reads = %v, want one message", reads) + } + for id, n := range reads { + if n != 1 { + t.Fatalf("message %d read %d times, want 1", id, n) + } + } + for _, m := range e.snd.sent { + if !strings.HasSuffix(string(m.Data), "Subject: hi\r\n\r\nbody\r\n") { + t.Fatalf("data = %q", m.Data) + } + } + // The cache lives for one run only: a later run reads again. + e.enqueue(t, 1) + e.w.RunOnce(context.Background()) + total := 0 + for _, n := range reads { + total += n + } + if total != 2 { + t.Fatalf("the next run must read its own body: %v", reads) + } +} + +func TestEachDistinctMessageIsReadOnce(t *testing.T) { + e := setup(t) + var calls int32 + e.w.readBody = func(id int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return e.s.OutboxData(id) } + for i := 0; i < 3; i++ { + e.enqueue(t, 2) + } + if st, _ := e.w.RunOnce(context.Background()); st.Sent != 6 { + t.Fatalf("sent %d", st.Sent) + } + if n := atomic.LoadInt32(&calls); n != 3 { + t.Fatalf("%d reads for 3 messages", n) + } +} + +func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) { + e := setup(t) + var calls int32 + e.w.readBody = func(int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return nil, store.ErrNotFound } + e.enqueue(t, 3) + st, _ := e.w.RunOnce(context.Background()) + if st.Dropped != 3 || e.snd.count() != 0 { + t.Fatalf("stats %+v", st) + } + if n := atomic.LoadInt32(&calls); n != 1 { + t.Fatalf("%d reads", n) + } +} diff --git a/internal/router/headers.go b/internal/router/headers.go index c9f5a3c..49a3585 100644 --- a/internal/router/headers.go +++ b/internal/router/headers.go @@ -7,6 +7,7 @@ import ( "io" "mime" "net/mail" + "strings" "github.com/emersion/go-message/textproto" @@ -23,9 +24,17 @@ func rewrite(l store.List, data []byte) ([]byte, error) { } // Never trust list headers coming from outside. - for _, k := range []string{"List-Id", "List-Post", "List-Unsubscribe", "List-Unsubscribe-Post", "Precedence"} { + for _, k := range []string{"List-Id", "List-Post", "List-Unsubscribe", "List-Unsubscribe-Post", "Precedence", + "List-Help", "List-Archive", "List-Owner", "List-Subscribe", + // The author's signature and authentication results do not survive our changes: they only confuse receivers. + "DKIM-Signature", "Authentication-Results"} { h.Del(k) } + for f := h.Fields(); f.Next(); { + if strings.HasPrefix(strings.ToLower(f.Key()), "arc-") { + f.Del() + } + } id := l.Name + "." + l.Domain // The phrase is RFC 2047 encoded when it is not ASCII; the stays plain ASCII (list names and domains are). diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index b59a5ed..ab9d060 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -585,3 +585,46 @@ func TestLeaveConfirmationWorksOnAClosedList(t *testing.T) { t.Fatal("alice must be gone") } } + +func TestIncomingListManagementAndAuthenticationHeadersAreStripped(t *testing.T) { + r := setup(t) + msg := "From: Alice \r\n" + + "List-Help: \r\nList-Archive: \r\n" + + "List-Owner: \r\nList-Subscribe: \r\n" + + "DKIM-Signature: v=1; a=rsa-sha256; d=example.com; s=x; h=from; bh=aa; b=bb\r\n" + + "Authentication-Results: mx.example.com; dkim=pass\r\n" + + "ARC-Seal: i=1; a=rsa-sha256; cv=none; d=example.com; s=x; b=cc\r\n" + + "ARC-Message-Signature: i=1; a=rsa-sha256; d=example.com; s=x; b=dd\r\n" + + "ARC-Authentication-Results: i=1; mx.example.com; dkim=pass\r\n" + + "Subject: hi\r\n\r\nx\r\n" + res, err := r.Route("alice@example.com", "dev@example.org", []byte(msg)) + if err != nil { + t.Fatal(err) + } + h := header(t, res.Data) + for _, k := range []string{"List-Help", "List-Archive", "List-Owner", "List-Subscribe", "DKIM-Signature", + "Authentication-Results", "ARC-Seal", "ARC-Message-Signature", "ARC-Authentication-Results"} { + if v := h.Get(k); v != "" { // Get canonicalizes the key + t.Errorf("%s survived: %v", k, v) + } + } + if h.Get("Subject") != "hi" { + t.Errorf("unrelated headers must stay: %v", h) + } +} + +func TestDroppingARCHeadersKeepsSimilarlyNamedOnes(t *testing.T) { + r := setup(t) + msg := "From: Alice \r\nARC-Seal: i=1\r\nArcade: keep\r\nX-ARC-Note: keep\r\nARC-Seal: i=2\r\n\r\nx\r\n" + res, err := r.Route("alice@example.com", "dev@example.org", []byte(msg)) + if err != nil { + t.Fatal(err) + } + h := header(t, res.Data) + if v := h["Arc-Seal"]; len(v) != 0 { + t.Errorf("every ARC-Seal must go: %v", v) + } + if h.Get("Arcade") != "keep" || h.Get("X-Arc-Note") != "keep" { + t.Errorf("only the ARC- prefix is dropped: %v", h) + } +}