parsimony/scale/scale_test.go
tgiachi 93a3edc58f
All checks were successful
ci / admin (push) Successful in 37s
ci / vuln (push) Successful in 48s
ci / go (push) Successful in 3m9s
chore: move the module to git.kmalloc.org and track cmd/parsimony
- Rename the Go module from codeberg.org/squidd/parsimony to
  git.kmalloc.org/squid/parsimony in go.mod and every import.
- Fix .gitignore: the bare `parsimony` pattern also ignored the cmd/parsimony
  folder, so the main package was never committed and a clean checkout could
  not be built. Anchor it to the root (`/parsimony`) and add cmd/parsimony.
2026-11-16 09:40:48 +01:00

199 lines
6.4 KiB
Go

//go:build scale
// Package scale measures how the daemon behaves with big lists and big archives.
//
// make scale
//
// It prints timings (run with -v); the assertions only catch pathological slowness.
package scale
import (
"context"
"fmt"
"io"
"net"
"os"
"path/filepath"
"sort"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/emersion/go-smtp"
"git.kmalloc.org/squid/parsimony/internal/archive"
"git.kmalloc.org/squid/parsimony/internal/outbound"
"git.kmalloc.org/squid/parsimony/internal/queue"
"git.kmalloc.org/squid/parsimony/internal/router"
"git.kmalloc.org/squid/parsimony/internal/store"
)
func openStore(t *testing.T) *store.Store {
t.Helper()
s, err := store.Open(filepath.Join(t.TempDir(), "scale.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
return s
}
func timed(t *testing.T, what string, f func()) time.Duration {
t.Helper()
start := time.Now()
f()
d := time.Since(start)
t.Logf("%-58s %v", what, d.Round(time.Millisecond))
return d
}
// countingRelay is an SMTP server that accepts and counts messages.
type countingRelay struct{ n atomic.Int64 }
func (c *countingRelay) NewSession(*smtp.Conn) (smtp.Session, error) { return &countSession{c}, nil }
type countSession struct{ c *countingRelay }
func (s *countSession) Reset() {}
func (s *countSession) Logout() error { return nil }
func (s *countSession) Mail(string, *smtp.MailOptions) error { return nil }
func (s *countSession) Rcpt(string, *smtp.RcptOptions) error { return nil }
func (s *countSession) Data(r io.Reader) error {
io.Copy(io.Discard, r)
s.c.n.Add(1)
return nil
}
func TestBigListFanOut(t *testing.T) {
const members = 10000
s := openStore(t)
l, _ := s.CreateList("big", "example.org", "Big")
timed(t, fmt.Sprintf("subscribe %d members", members), func() {
for i := 0; i < members; i++ {
if err := s.AddMember(l.ID, fmt.Sprintf("user%d@example.com", i), ""); err != nil {
t.Fatal(err)
}
}
})
s.AddMember(l.ID, "author@example.com", "Author")
rt := router.New(s)
post := []byte("From: Author <author@example.com>\r\nSubject: Big news\r\nMessage-ID: <big1@x>\r\n\r\n" + strings.Repeat(strings.Repeat("x", 70)+"\r\n", 30))
var res router.Result
timed(t, "route a post (checks, archive, header rewrite)", func() {
var err error
if res, err = rt.Route("author@example.com", "big@example.org", post); err != nil {
t.Fatal(err)
}
})
if d := timed(t, fmt.Sprintf("queue it for %d recipients (one transaction)", members+1), func() {
if err := rt.Deliver(res); err != nil {
t.Fatal(err)
}
}); d > 30*time.Second {
t.Fatalf("queueing took %v", d)
}
relay := &countingRelay{}
srv := smtp.NewServer(relay)
ln, _ := net.Listen("tcp", "127.0.0.1:0")
go srv.Serve(ln)
defer srv.Close()
w := &queue.Worker{Store: s, Sender: &outbound.SMTPRelay{Addr: ln.Addr().String()}, Concurrency: 4, Batch: 200}
var sent int
d := timed(t, fmt.Sprintf("send %d messages to a local relay (4 at a time)", members+1), func() {
for {
st, err := w.RunOnce(context.Background())
if err != nil {
t.Fatal(err)
}
sent += st.Sent
if st.Sent == 0 {
return
}
}
})
if sent != members+1 || relay.n.Load() != int64(members+1) {
t.Fatalf("sent %d, relay got %d", sent, relay.n.Load())
}
t.Logf("%-58s %.0f messages/second", "throughput with a local relay", float64(sent)/d.Seconds())
if d > 5*time.Minute {
t.Fatalf("the queue drained too slowly: %v", d)
}
}
func TestBigArchive(t *testing.T) {
const messages = 50000
s := openStore(t)
l, _ := s.CreateList("big", "example.org", "Big")
words := []string{"deploy", "kubernetes", "release", "database", "backup", "outage", "billing", "migration", "latency", "invoice"}
var b strings.Builder
for i := 0; i < messages; i++ {
reply := ""
if i%4 != 0 {
reply = fmt.Sprintf("In-Reply-To: <m%d@x>\n", i-1)
}
fmt.Fprintf(&b, "From user%d@example.com Tue Jan 2 15:04:05 2024\nFrom: User %d <user%d@example.com>\nSubject: %s %s %d\nMessage-ID: <m%d@x>\n%sDate: Mon, 01 Jan 2024 10:00:00 +0000\n\n%s %s %s %s\nsome more text to make the body a little longer than the subject.\n\n",
i%500, i%500, i%500, words[i%10], words[(i/10)%10], i, i, reply, words[i%10], words[(i/7)%10], words[(i/13)%10], words[(i/17)%10])
}
mbox := b.String()
t.Logf("%-58s %.1f MiB", fmt.Sprintf("generated mbox with %d messages", messages), float64(len(mbox))/(1<<20))
var sum archive.ImportSummary
d := timed(t, fmt.Sprintf("import %d messages (parse, batch insert, FTS, rethread)", messages), func() {
var err error
if sum, err = archive.ImportMbox(s, l.ID, strings.NewReader(mbox), time.Now()); err != nil {
t.Fatal(err)
}
})
if sum.Imported != messages || sum.Failed != 0 {
t.Fatalf("summary: %+v", sum)
}
t.Logf("%-58s %.0f messages/second", "import throughput", float64(messages)/d.Seconds())
if d > 10*time.Minute {
t.Fatalf("the import took %v", d)
}
timed(t, "import the same file again (all duplicates)", func() {
again, _ := archive.ImportMbox(s, l.ID, strings.NewReader(mbox), time.Now())
if again.Duplicates != messages {
t.Fatalf("%+v", again)
}
})
timed(t, fmt.Sprintf("rethread %d messages", messages), func() {
if _, err := s.Rethread(l.ID); err != nil {
t.Fatal(err)
}
})
latency := func(f func()) (median, worst time.Duration) {
var all []time.Duration
for i := 0; i < 20; i++ {
start := time.Now()
f()
all = append(all, time.Since(start))
}
sort.Slice(all, func(i, j int) bool { return all[i] < all[j] })
return all[len(all)/2], all[len(all)-1]
}
for _, q := range []string{"kubernetes", "deploy outage", "invoice billing latency", "zzzznotfound"} {
med, worst := latency(func() {
if _, _, err := s.SearchMessages(l.ID, q, 25, 0); err != nil {
t.Fatal(err)
}
})
t.Logf("%-58s median %v, worst %v", fmt.Sprintf("search %q (one page of 25)", q), med.Round(time.Microsecond), worst.Round(time.Microsecond))
if med > 2*time.Second {
t.Fatalf("search %q is too slow: %v", q, med)
}
}
med, worst := latency(func() { s.ListMessages(l.ID, 25, 25*1000) })
t.Logf("%-58s median %v, worst %v", "list page 1000 of the archive", med.Round(time.Microsecond), worst.Round(time.Microsecond))
med, worst = latency(func() { s.ThreadMessages(l.ID, 1) })
t.Logf("%-58s median %v, worst %v", "read a thread", med.Round(time.Microsecond), worst.Round(time.Microsecond))
if fi, err := os.Stat(filepath.Join(os.TempDir())); err == nil {
_ = fi
}
}