perf(queue): purge old failures at most every 5 minutes and read each body once per run
- RunOnce purged failed deliveries (a full scan) on every batch of 50: the Worker now does it at most once every 5 minutes, by the store clock; a failed purge is retried on the next run. - Every recipient re-read the whole message from SQLite: a per-run map msgID -> body, filled once per message even under concurrency, is dropped when the run ends. - Tests count purges across batches and after the interval, and body reads per message (including a missing body) through unexported hooks.
This commit is contained in:
parent
b09ee2e419
commit
d9012cc13a
2 changed files with 170 additions and 6 deletions
|
|
@ -22,6 +22,9 @@ const (
|
|||
defaultBatch = 50
|
||||
defaultTimeout = time.Minute
|
||||
defaultInterval = 5 * time.Second
|
||||
|
||||
// purgeEvery is the least time between two scans that forget old failed deliveries.
|
||||
purgeEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour}
|
||||
|
|
@ -61,6 +64,53 @@ type Worker struct {
|
|||
wakeOnce sync.Once
|
||||
wake chan struct{}
|
||||
runMu sync.Mutex
|
||||
|
||||
lastPurge time.Time // under runMu; zero until the first purge
|
||||
// Hooks for tests; nil means the Store methods.
|
||||
purge func() (int, error)
|
||||
readBody func(msgID int64) ([]byte, error)
|
||||
}
|
||||
|
||||
// bodies reads each message body once per run, however many recipients share it.
|
||||
type bodies struct {
|
||||
read func(msgID int64) ([]byte, error)
|
||||
mu sync.Mutex
|
||||
m map[int64]*body
|
||||
}
|
||||
|
||||
type body struct {
|
||||
once sync.Once
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (b *bodies) get(msgID int64) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
e := b.m[msgID]
|
||||
if e == nil {
|
||||
e = &body{}
|
||||
b.m[msgID] = e
|
||||
}
|
||||
b.mu.Unlock()
|
||||
e.once.Do(func() { e.data, e.err = b.read(msgID) })
|
||||
return e.data, e.err
|
||||
}
|
||||
|
||||
// purgeOld forgets old failed deliveries, at most once every purgeEvery.
|
||||
func (w *Worker) purgeOld() {
|
||||
now := w.Store.Now()
|
||||
if !w.lastPurge.IsZero() && now.Sub(w.lastPurge) < purgeEvery {
|
||||
return
|
||||
}
|
||||
purge := w.purge
|
||||
if purge == nil {
|
||||
purge = w.Store.PurgeFailedOutbox
|
||||
}
|
||||
if _, err := purge(); err != nil {
|
||||
log.Printf("queue: purge old failures: %v", err)
|
||||
return // try again on the next run
|
||||
}
|
||||
w.lastPurge = now
|
||||
}
|
||||
|
||||
func (w *Worker) wakeCh() chan struct{} {
|
||||
|
|
@ -125,14 +175,18 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
|||
if conc <= 0 {
|
||||
conc = defaultConcurrency
|
||||
}
|
||||
if _, err := w.Store.PurgeFailedOutbox(); err != nil {
|
||||
log.Printf("queue: purge old failures: %v", err)
|
||||
}
|
||||
w.purgeOld()
|
||||
due, err := w.Store.DueOutbox(batch)
|
||||
if err != nil {
|
||||
return Stats{}, err
|
||||
}
|
||||
|
||||
read := w.readBody
|
||||
if read == nil {
|
||||
read = w.Store.OutboxData
|
||||
}
|
||||
cache := &bodies{read: read, m: map[int64]*body{}}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
stats Stats
|
||||
|
|
@ -148,7 +202,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
|||
go func(it store.OutboxItem) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
o := w.deliver(ctx, it)
|
||||
o := w.deliver(ctx, cache, it)
|
||||
mu.Lock()
|
||||
stats.Sent += o.Sent
|
||||
stats.Retried += o.Retried
|
||||
|
|
@ -162,8 +216,8 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
|||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
|
||||
data, err := w.Store.OutboxData(it.MsgID)
|
||||
func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats {
|
||||
data, err := cache.get(it.MsgID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
|
|
|
|||
|
|
@ -558,3 +558,113 @@ func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) {
|
|||
t.Fatalf("every delivery stays listed as failed: %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPurgeRunsOncePerIntervalNotOnEveryBatch(t *testing.T) {
|
||||
e := setup(t)
|
||||
var purges int32
|
||||
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, nil }
|
||||
e.w.Batch = 1
|
||||
e.enqueue(t, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 1 {
|
||||
t.Fatalf("run %d: %+v %v", i, st, err)
|
||||
}
|
||||
}
|
||||
if n := atomic.LoadInt32(&purges); n != 1 {
|
||||
t.Fatalf("purge ran %d times in 3 batches, want 1", n)
|
||||
}
|
||||
e.clk.set(t0.Add(purgeEvery - time.Second))
|
||||
e.w.RunOnce(context.Background())
|
||||
if n := atomic.LoadInt32(&purges); n != 1 {
|
||||
t.Fatalf("purge ran again before the interval: %d", n)
|
||||
}
|
||||
e.clk.set(t0.Add(purgeEvery))
|
||||
e.w.RunOnce(context.Background())
|
||||
if n := atomic.LoadInt32(&purges); n != 2 {
|
||||
t.Fatalf("purge must run again after the interval: %d", n)
|
||||
}
|
||||
e.w.RunOnce(context.Background())
|
||||
if n := atomic.LoadInt32(&purges); n != 2 {
|
||||
t.Fatalf("interval counts from the last purge: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedPurgeIsRetriedOnTheNextRun(t *testing.T) {
|
||||
e := setup(t)
|
||||
var purges int32
|
||||
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, errors.New("db busy") }
|
||||
e.w.RunOnce(context.Background())
|
||||
e.w.RunOnce(context.Background())
|
||||
if n := atomic.LoadInt32(&purges); n != 2 {
|
||||
t.Fatalf("a failed purge must not delay the next attempt: %d runs", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneBodyReadServesEveryRecipientOfAMessage(t *testing.T) {
|
||||
e := setup(t)
|
||||
reads := map[int64]int{}
|
||||
var mu sync.Mutex
|
||||
e.w.readBody = func(id int64) ([]byte, error) {
|
||||
mu.Lock()
|
||||
reads[id]++
|
||||
mu.Unlock()
|
||||
return e.s.OutboxData(id)
|
||||
}
|
||||
e.enqueue(t, 6)
|
||||
e.w.Concurrency = 6
|
||||
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 6 {
|
||||
t.Fatalf("run: %+v %v", st, err)
|
||||
}
|
||||
if len(reads) != 1 {
|
||||
t.Fatalf("reads = %v, want one message", reads)
|
||||
}
|
||||
for id, n := range reads {
|
||||
if n != 1 {
|
||||
t.Fatalf("message %d read %d times, want 1", id, n)
|
||||
}
|
||||
}
|
||||
for _, m := range e.snd.sent {
|
||||
if !strings.HasSuffix(string(m.Data), "Subject: hi\r\n\r\nbody\r\n") {
|
||||
t.Fatalf("data = %q", m.Data)
|
||||
}
|
||||
}
|
||||
// The cache lives for one run only: a later run reads again.
|
||||
e.enqueue(t, 1)
|
||||
e.w.RunOnce(context.Background())
|
||||
total := 0
|
||||
for _, n := range reads {
|
||||
total += n
|
||||
}
|
||||
if total != 2 {
|
||||
t.Fatalf("the next run must read its own body: %v", reads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachDistinctMessageIsReadOnce(t *testing.T) {
|
||||
e := setup(t)
|
||||
var calls int32
|
||||
e.w.readBody = func(id int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return e.s.OutboxData(id) }
|
||||
for i := 0; i < 3; i++ {
|
||||
e.enqueue(t, 2)
|
||||
}
|
||||
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 6 {
|
||||
t.Fatalf("sent %d", st.Sent)
|
||||
}
|
||||
if n := atomic.LoadInt32(&calls); n != 3 {
|
||||
t.Fatalf("%d reads for 3 messages", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) {
|
||||
e := setup(t)
|
||||
var calls int32
|
||||
e.w.readBody = func(int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return nil, store.ErrNotFound }
|
||||
e.enqueue(t, 3)
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 3 || e.snd.count() != 0 {
|
||||
t.Fatalf("stats %+v", st)
|
||||
}
|
||||
if n := atomic.LoadInt32(&calls); n != 1 {
|
||||
t.Fatalf("%d reads", n)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue