diff --git a/Makefile b/Makefile index f16d7fb..c053f09 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all admin test build e2e clean fmt-check lint vuln go-ci admin-ci ci +.PHONY: all admin test build e2e scale clean fmt-check lint vuln go-ci admin-ci ci all: admin build @@ -38,6 +38,10 @@ ci: go-ci admin-ci build e2e: go test -tags e2e -count=1 -timeout 15m -v ./e2e +# How the daemon behaves with a 10000-member list and a 50000-message archive (prints timings). +scale: + go test -tags scale -count=1 -timeout 30m -v ./scale + clean: rm -f parsimony node admin/scripts/clean.mjs diff --git a/README.md b/README.md index 836d04e..bf41b11 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,21 @@ lists.example.org lmtp:inet:127.0.0.1:2424 - The API and the admin UI use one bearer token and no TLS: put them behind a reverse proxy with TLS and keep the `api` listener private. +## Scale + +Measured on one machine with `make scale` (SQLite file on a local disk, relay on localhost; your numbers will differ): + +| | | +|---|---| +| Queue a post for a 10 001-member list (one transaction) | 0.12 s | +| Send it through a local relay, 4 deliveries at a time | about 460 messages/s (22 s for the whole list) | +| Import a 50 000-message mbox (14 MiB) | 2.9 s, about 17 000 messages/s | +| Search a 50 000-message archive (one page of 25) | 6 to 15 ms | +| Open page 1 000 of the archive, read a thread | about 4 ms, about 9 ms | +| Subscribe members one at a time (`member add`, no batching) | about 1.9 ms each at the store, plus one HTTP request | + +Sending is bounded by the SMTP round trips to your relay: raise `relay.concurrency` for big lists. + ## Licence Parsimony is free software, released under the GNU General Public License, version 3 or (at your option) any later diff --git a/scale/scale_test.go b/scale/scale_test.go new file mode 100644 index 0000000..145b0d0 --- /dev/null +++ b/scale/scale_test.go @@ -0,0 +1,199 @@ +//go:build scale + +// Package scale measures how the daemon behaves with big lists and big archives. +// +// make scale +// +// It prints timings (run with -v); the assertions only catch pathological slowness. +package scale + +import ( + "context" + "fmt" + "io" + "net" + "os" + "path/filepath" + "sort" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/emersion/go-smtp" + + "codeberg.org/squidd/parsimony/internal/archive" + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/queue" + "codeberg.org/squidd/parsimony/internal/router" + "codeberg.org/squidd/parsimony/internal/store" +) + +func openStore(t *testing.T) *store.Store { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "scale.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func timed(t *testing.T, what string, f func()) time.Duration { + t.Helper() + start := time.Now() + f() + d := time.Since(start) + t.Logf("%-58s %v", what, d.Round(time.Millisecond)) + return d +} + +// countingRelay is an SMTP server that accepts and counts messages. +type countingRelay struct{ n atomic.Int64 } + +func (c *countingRelay) NewSession(*smtp.Conn) (smtp.Session, error) { return &countSession{c}, nil } + +type countSession struct{ c *countingRelay } + +func (s *countSession) Reset() {} +func (s *countSession) Logout() error { return nil } +func (s *countSession) Mail(string, *smtp.MailOptions) error { return nil } +func (s *countSession) Rcpt(string, *smtp.RcptOptions) error { return nil } +func (s *countSession) Data(r io.Reader) error { + io.Copy(io.Discard, r) + s.c.n.Add(1) + return nil +} + +func TestBigListFanOut(t *testing.T) { + const members = 10000 + s := openStore(t) + l, _ := s.CreateList("big", "example.org", "Big") + timed(t, fmt.Sprintf("subscribe %d members", members), func() { + for i := 0; i < members; i++ { + if err := s.AddMember(l.ID, fmt.Sprintf("user%d@example.com", i), ""); err != nil { + t.Fatal(err) + } + } + }) + s.AddMember(l.ID, "author@example.com", "Author") + rt := router.New(s) + post := []byte("From: Author \r\nSubject: Big news\r\nMessage-ID: \r\n\r\n" + strings.Repeat(strings.Repeat("x", 70)+"\r\n", 30)) + + var res router.Result + timed(t, "route a post (checks, archive, header rewrite)", func() { + var err error + if res, err = rt.Route("author@example.com", "big@example.org", post); err != nil { + t.Fatal(err) + } + }) + if d := timed(t, fmt.Sprintf("queue it for %d recipients (one transaction)", members+1), func() { + if err := rt.Deliver(res); err != nil { + t.Fatal(err) + } + }); d > 30*time.Second { + t.Fatalf("queueing took %v", d) + } + + relay := &countingRelay{} + srv := smtp.NewServer(relay) + ln, _ := net.Listen("tcp", "127.0.0.1:0") + go srv.Serve(ln) + defer srv.Close() + w := &queue.Worker{Store: s, Sender: &outbound.SMTPRelay{Addr: ln.Addr().String()}, Concurrency: 4, Batch: 200} + var sent int + d := timed(t, fmt.Sprintf("send %d messages to a local relay (4 at a time)", members+1), func() { + for { + st, err := w.RunOnce(context.Background()) + if err != nil { + t.Fatal(err) + } + sent += st.Sent + if st.Sent == 0 { + return + } + } + }) + if sent != members+1 || relay.n.Load() != int64(members+1) { + t.Fatalf("sent %d, relay got %d", sent, relay.n.Load()) + } + t.Logf("%-58s %.0f messages/second", "throughput with a local relay", float64(sent)/d.Seconds()) + if d > 5*time.Minute { + t.Fatalf("the queue drained too slowly: %v", d) + } +} + +func TestBigArchive(t *testing.T) { + const messages = 50000 + s := openStore(t) + l, _ := s.CreateList("big", "example.org", "Big") + words := []string{"deploy", "kubernetes", "release", "database", "backup", "outage", "billing", "migration", "latency", "invoice"} + var b strings.Builder + for i := 0; i < messages; i++ { + reply := "" + if i%4 != 0 { + reply = fmt.Sprintf("In-Reply-To: \n", i-1) + } + fmt.Fprintf(&b, "From user%d@example.com Tue Jan 2 15:04:05 2024\nFrom: User %d \nSubject: %s %s %d\nMessage-ID: \n%sDate: Mon, 01 Jan 2024 10:00:00 +0000\n\n%s %s %s %s\nsome more text to make the body a little longer than the subject.\n\n", + i%500, i%500, i%500, words[i%10], words[(i/10)%10], i, i, reply, words[i%10], words[(i/7)%10], words[(i/13)%10], words[(i/17)%10]) + } + mbox := b.String() + t.Logf("%-58s %.1f MiB", fmt.Sprintf("generated mbox with %d messages", messages), float64(len(mbox))/(1<<20)) + + var sum archive.ImportSummary + d := timed(t, fmt.Sprintf("import %d messages (parse, batch insert, FTS, rethread)", messages), func() { + var err error + if sum, err = archive.ImportMbox(s, l.ID, strings.NewReader(mbox), time.Now()); err != nil { + t.Fatal(err) + } + }) + if sum.Imported != messages || sum.Failed != 0 { + t.Fatalf("summary: %+v", sum) + } + t.Logf("%-58s %.0f messages/second", "import throughput", float64(messages)/d.Seconds()) + if d > 10*time.Minute { + t.Fatalf("the import took %v", d) + } + + timed(t, "import the same file again (all duplicates)", func() { + again, _ := archive.ImportMbox(s, l.ID, strings.NewReader(mbox), time.Now()) + if again.Duplicates != messages { + t.Fatalf("%+v", again) + } + }) + timed(t, fmt.Sprintf("rethread %d messages", messages), func() { + if _, err := s.Rethread(l.ID); err != nil { + t.Fatal(err) + } + }) + + latency := func(f func()) (median, worst time.Duration) { + var all []time.Duration + for i := 0; i < 20; i++ { + start := time.Now() + f() + all = append(all, time.Since(start)) + } + sort.Slice(all, func(i, j int) bool { return all[i] < all[j] }) + return all[len(all)/2], all[len(all)-1] + } + for _, q := range []string{"kubernetes", "deploy outage", "invoice billing latency", "zzzznotfound"} { + med, worst := latency(func() { + if _, _, err := s.SearchMessages(l.ID, q, 25, 0); err != nil { + t.Fatal(err) + } + }) + t.Logf("%-58s median %v, worst %v", fmt.Sprintf("search %q (one page of 25)", q), med.Round(time.Microsecond), worst.Round(time.Microsecond)) + if med > 2*time.Second { + t.Fatalf("search %q is too slow: %v", q, med) + } + } + med, worst := latency(func() { s.ListMessages(l.ID, 25, 25*1000) }) + t.Logf("%-58s median %v, worst %v", "list page 1000 of the archive", med.Round(time.Microsecond), worst.Round(time.Microsecond)) + med, worst = latency(func() { s.ThreadMessages(l.ID, 1) }) + t.Logf("%-58s median %v, worst %v", "read a thread", med.Round(time.Microsecond), worst.Round(time.Microsecond)) + + if fi, err := os.Stat(filepath.Join(os.TempDir())); err == nil { + _ = fi + } +}