//go:build scale // Package scale measures how the daemon behaves with big lists and big archives. // // make scale // // It prints timings (run with -v); the assertions only catch pathological slowness. package scale import ( "context" "fmt" "io" "net" "os" "path/filepath" "sort" "strings" "sync/atomic" "testing" "time" "github.com/emersion/go-smtp" "git.kmalloc.org/squid/parsimony/internal/archive" "git.kmalloc.org/squid/parsimony/internal/outbound" "git.kmalloc.org/squid/parsimony/internal/queue" "git.kmalloc.org/squid/parsimony/internal/router" "git.kmalloc.org/squid/parsimony/internal/store" ) func openStore(t *testing.T) *store.Store { t.Helper() s, err := store.Open(filepath.Join(t.TempDir(), "scale.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { s.Close() }) return s } func timed(t *testing.T, what string, f func()) time.Duration { t.Helper() start := time.Now() f() d := time.Since(start) t.Logf("%-58s %v", what, d.Round(time.Millisecond)) return d } // countingRelay is an SMTP server that accepts and counts messages. type countingRelay struct{ n atomic.Int64 } func (c *countingRelay) NewSession(*smtp.Conn) (smtp.Session, error) { return &countSession{c}, nil } type countSession struct{ c *countingRelay } func (s *countSession) Reset() {} func (s *countSession) Logout() error { return nil } func (s *countSession) Mail(string, *smtp.MailOptions) error { return nil } func (s *countSession) Rcpt(string, *smtp.RcptOptions) error { return nil } func (s *countSession) Data(r io.Reader) error { io.Copy(io.Discard, r) s.c.n.Add(1) return nil } func TestBigListFanOut(t *testing.T) { const members = 10000 s := openStore(t) l, _ := s.CreateList("big", "example.org", "Big") timed(t, fmt.Sprintf("subscribe %d members", members), func() { for i := 0; i < members; i++ { if err := s.AddMember(l.ID, fmt.Sprintf("user%d@example.com", i), ""); err != nil { t.Fatal(err) } } }) s.AddMember(l.ID, "author@example.com", "Author") rt := router.New(s) post := []byte("From: Author \r\nSubject: Big news\r\nMessage-ID: \r\n\r\n" + strings.Repeat(strings.Repeat("x", 70)+"\r\n", 30)) var res router.Result timed(t, "route a post (checks, archive, header rewrite)", func() { var err error if res, err = rt.Route("author@example.com", "big@example.org", post); err != nil { t.Fatal(err) } }) if d := timed(t, fmt.Sprintf("queue it for %d recipients (one transaction)", members+1), func() { if err := rt.Deliver(res); err != nil { t.Fatal(err) } }); d > 30*time.Second { t.Fatalf("queueing took %v", d) } relay := &countingRelay{} srv := smtp.NewServer(relay) ln, _ := net.Listen("tcp", "127.0.0.1:0") go srv.Serve(ln) defer srv.Close() w := &queue.Worker{Store: s, Sender: &outbound.SMTPRelay{Addr: ln.Addr().String()}, Concurrency: 4, Batch: 200} var sent int d := timed(t, fmt.Sprintf("send %d messages to a local relay (4 at a time)", members+1), func() { for { st, err := w.RunOnce(context.Background()) if err != nil { t.Fatal(err) } sent += st.Sent if st.Sent == 0 { return } } }) if sent != members+1 || relay.n.Load() != int64(members+1) { t.Fatalf("sent %d, relay got %d", sent, relay.n.Load()) } t.Logf("%-58s %.0f messages/second", "throughput with a local relay", float64(sent)/d.Seconds()) if d > 5*time.Minute { t.Fatalf("the queue drained too slowly: %v", d) } } func TestBigArchive(t *testing.T) { const messages = 50000 s := openStore(t) l, _ := s.CreateList("big", "example.org", "Big") words := []string{"deploy", "kubernetes", "release", "database", "backup", "outage", "billing", "migration", "latency", "invoice"} var b strings.Builder for i := 0; i < messages; i++ { reply := "" if i%4 != 0 { reply = fmt.Sprintf("In-Reply-To: \n", i-1) } fmt.Fprintf(&b, "From user%d@example.com Tue Jan 2 15:04:05 2024\nFrom: User %d \nSubject: %s %s %d\nMessage-ID: \n%sDate: Mon, 01 Jan 2024 10:00:00 +0000\n\n%s %s %s %s\nsome more text to make the body a little longer than the subject.\n\n", i%500, i%500, i%500, words[i%10], words[(i/10)%10], i, i, reply, words[i%10], words[(i/7)%10], words[(i/13)%10], words[(i/17)%10]) } mbox := b.String() t.Logf("%-58s %.1f MiB", fmt.Sprintf("generated mbox with %d messages", messages), float64(len(mbox))/(1<<20)) var sum archive.ImportSummary d := timed(t, fmt.Sprintf("import %d messages (parse, batch insert, FTS, rethread)", messages), func() { var err error if sum, err = archive.ImportMbox(s, l.ID, strings.NewReader(mbox), time.Now()); err != nil { t.Fatal(err) } }) if sum.Imported != messages || sum.Failed != 0 { t.Fatalf("summary: %+v", sum) } t.Logf("%-58s %.0f messages/second", "import throughput", float64(messages)/d.Seconds()) if d > 10*time.Minute { t.Fatalf("the import took %v", d) } timed(t, "import the same file again (all duplicates)", func() { again, _ := archive.ImportMbox(s, l.ID, strings.NewReader(mbox), time.Now()) if again.Duplicates != messages { t.Fatalf("%+v", again) } }) timed(t, fmt.Sprintf("rethread %d messages", messages), func() { if _, err := s.Rethread(l.ID); err != nil { t.Fatal(err) } }) latency := func(f func()) (median, worst time.Duration) { var all []time.Duration for i := 0; i < 20; i++ { start := time.Now() f() all = append(all, time.Since(start)) } sort.Slice(all, func(i, j int) bool { return all[i] < all[j] }) return all[len(all)/2], all[len(all)-1] } for _, q := range []string{"kubernetes", "deploy outage", "invoice billing latency", "zzzznotfound"} { med, worst := latency(func() { if _, _, err := s.SearchMessages(l.ID, q, 25, 0); err != nil { t.Fatal(err) } }) t.Logf("%-58s median %v, worst %v", fmt.Sprintf("search %q (one page of 25)", q), med.Round(time.Microsecond), worst.Round(time.Microsecond)) if med > 2*time.Second { t.Fatalf("search %q is too slow: %v", q, med) } } med, worst := latency(func() { s.ListMessages(l.ID, 25, 25*1000) }) t.Logf("%-58s median %v, worst %v", "list page 1000 of the archive", med.Round(time.Microsecond), worst.Round(time.Microsecond)) med, worst = latency(func() { s.ThreadMessages(l.ID, 1) }) t.Logf("%-58s median %v, worst %v", "read a thread", med.Round(time.Microsecond), worst.Round(time.Microsecond)) if fi, err := os.Stat(filepath.Join(os.TempDir())); err == nil { _ = fi } }