parsimony/internal/ingest/ingest_test.go
tgiachi 93a3edc58f
All checks were successful
ci / admin (push) Successful in 37s
ci / vuln (push) Successful in 48s
ci / go (push) Successful in 3m9s
chore: move the module to git.kmalloc.org and track cmd/parsimony
- Rename the Go module from codeberg.org/squidd/parsimony to
  git.kmalloc.org/squid/parsimony in go.mod and every import.
- Fix .gitignore: the bare `parsimony` pattern also ignored the cmd/parsimony
  folder, so the main package was never committed and a clean checkout could
  not be built. Anchor it to the root (`/parsimony`) and add cmd/parsimony.
2026-11-16 09:40:48 +01:00

353 lines
10 KiB
Go

package ingest
import (
"context"
"errors"
"net"
"strings"
"testing"
"time"
"github.com/emersion/go-smtp"
"git.kmalloc.org/squid/parsimony/internal/outbound"
"git.kmalloc.org/squid/parsimony/internal/queue"
"git.kmalloc.org/squid/parsimony/internal/router"
"git.kmalloc.org/squid/parsimony/internal/store"
)
const msg = "From: Alice <alice@example.com>\r\nSubject: hi\r\n\r\nbody\r\n"
// stack is the LMTP server with a real queue worker in front of a fake relay.
type stack struct {
addr string
relay *outbound.Fake
store *store.Store
}
func newStack(t *testing.T, sender outbound.Sender) *stack {
t.Helper()
s, err := store.Open(":memory:")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
l, _ := s.CreateList("dev", "example.org", "Dev")
s.AddMember(l.ID, "alice@example.com", "Alice")
s.AddMember(l.ID, "bob@example.com", "Bob")
relay := &outbound.Fake{}
if sender == nil {
sender = relay
}
rt := router.New(s)
w := &queue.Worker{Store: s, Sender: sender, Interval: 10 * time.Millisecond}
rt.OnEnqueue = w.Wake
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { w.Run(ctx); close(done) }()
t.Cleanup(func() { cancel(); <-done })
srv := NewServer(rt)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
go srv.Serve(ln)
t.Cleanup(func() { srv.Close() })
return &stack{addr: ln.Addr().String(), relay: relay, store: s}
}
// sent waits until the relay has received n messages and returns them.
func (st *stack) sent(t *testing.T, n int) []outbound.Sent {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for {
got := st.relay.Messages()
if len(got) >= n {
return got
}
if time.Now().After(deadline) {
t.Fatalf("relay got %d messages, want %d", len(got), n)
}
time.Sleep(5 * time.Millisecond)
}
}
// queued is how many deliveries still wait in the outbox.
func (st *stack) queued(t *testing.T) int {
t.Helper()
stats, err := st.store.OutboxStats()
if err != nil {
t.Fatal(err)
}
return stats.Pending
}
// deliver sends msg over LMTP and returns the per-recipient errors.
func deliver(t *testing.T, addr, from string, rcpts []string) map[string]error {
t.Helper()
conn, err := net.Dial("tcp", addr)
if err != nil {
t.Fatal(err)
}
c := smtp.NewClientLMTP(conn)
defer c.Close()
if err := c.Hello("localhost"); err != nil {
t.Fatal(err)
}
if err := c.Mail(from, nil); err != nil {
t.Fatal(err)
}
for _, r := range rcpts {
if err := c.Rcpt(r, nil); err != nil {
t.Fatal(err)
}
}
w, err := c.Data()
if err != nil {
t.Fatal(err)
}
w.Write([]byte(msg))
resp, err := w.CloseWithLMTPResponse()
out := map[string]error{}
var de smtp.LMTPDataError
if errors.As(err, &de) {
for k, v := range de {
out[k] = v
}
} else if err != nil {
t.Fatal(err)
}
for k := range resp {
out[k] = nil
}
return out
}
func TestDeliverFansOut(t *testing.T) {
st := newStack(t, nil)
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
if res["dev@example.org"] != nil {
t.Fatalf("unexpected error: %v", res)
}
got := st.sent(t, 2)
if len(got) != 2 {
t.Fatalf("want one send per member, got %d", len(got))
}
m := got[0]
if len(m.To) != 1 {
t.Errorf("recipients = %v", m.To)
}
if !strings.HasPrefix(m.From, "dev+bounce-") {
t.Errorf("envelope from = %q", m.From)
}
if !strings.Contains(string(m.Data), "List-Id: Dev <dev.example.org>") {
t.Errorf("missing List-Id: %q", m.Data)
}
if !strings.Contains(string(m.Data), "List-Unsubscribe: <mailto:dev+leave@example.org>") {
t.Errorf("missing per-recipient unsubscribe header: %q", m.Data)
}
}
func TestNonMemberRejectedPermanently(t *testing.T) {
st := newStack(t, nil)
res := deliver(t, st.addr, "mallory@evil.com", []string{"dev@example.org"})
var se *smtp.SMTPError
if !errors.As(res["dev@example.org"], &se) || se.Code != 550 {
t.Fatalf("want 550, got %v", res["dev@example.org"])
}
if st.queued(t) != 0 {
t.Fatal("nothing may be queued")
}
}
func TestUnknownListRejected(t *testing.T) {
st := newStack(t, nil)
res := deliver(t, st.addr, "alice@example.com", []string{"nope@example.org"})
var se *smtp.SMTPError
if !errors.As(res["nope@example.org"], &se) || se.Code != 550 {
t.Fatalf("want 550, got %v", res["nope@example.org"])
}
}
func TestMixedRecipients(t *testing.T) {
st := newStack(t, nil)
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"})
if res["dev@example.org"] != nil || res["nope@example.org"] == nil {
t.Fatalf("results: %v", res)
}
st.sent(t, 2)
}
// down is a relay that refuses everything.
type down struct{}
func (down) Send(context.Context, string, []string, []byte) error { return errors.New("relay down") }
func TestRelayDownStillAcceptsAndKeepsTheMailQueued(t *testing.T) {
st := newStack(t, down{})
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
if res["dev@example.org"] != nil {
t.Fatalf("a queued post is accepted even when the relay is down: %v", res)
}
// The worker tries and schedules retries; nothing is lost.
deadline := time.Now().Add(3 * time.Second)
for {
stats, _ := st.store.OutboxStats()
if stats.Pending == 2 && stats.Retrying == 2 {
break
}
if time.Now().After(deadline) {
t.Fatalf("queue: %+v", stats)
}
time.Sleep(5 * time.Millisecond)
}
}
func TestQueueFailureIsTemporary(t *testing.T) {
st := newStack(t, nil)
st.store.Close() // the database goes away under the server
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
var se *smtp.SMTPError
if !errors.As(res["dev@example.org"], &se) || se.Code != 451 {
t.Fatalf("want 451 so the MTA retries, got %v", res["dev@example.org"])
}
}
func TestJoinCommandOverLMTP(t *testing.T) {
st := newStack(t, nil)
res := deliver(t, st.addr, "carol@x.com", []string{"dev+join@example.org"})
if res["dev+join@example.org"] != nil {
t.Fatalf("join: %v", res)
}
got := st.sent(t, 1)
if len(got[0].To) != 1 || got[0].To[0] != "carol@x.com" {
t.Fatalf("confirmation mail: %+v", got)
}
}
func TestBadTokenRejected(t *testing.T) {
st := newStack(t, nil)
res := deliver(t, st.addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"})
var se *smtp.SMTPError
if !errors.As(res["dev+confirm-deadbeef@example.org"], &se) || se.Code != 550 {
t.Fatalf("want 550, got %v", res)
}
}
func TestModeratedPostIsAcceptedButNotQueued(t *testing.T) {
st := newStack(t, nil)
l, _ := st.store.FindList("dev@example.org")
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
res := deliver(t, st.addr, "stranger@x.com", []string{"dev@example.org"})
if res["dev@example.org"] != nil {
t.Fatalf("held mail must be accepted: %v", res)
}
if st.queued(t) != 0 {
t.Fatal("nothing may be queued for delivery")
}
if h, _ := st.store.ListHeld(l.ID); len(h) != 1 {
t.Fatalf("held = %d", len(h))
}
}
func TestAutoReplyAndLoopsAreAcceptedButNotDelivered(t *testing.T) {
st := newStack(t, nil)
for _, h := range []string{"Auto-Submitted: auto-replied\r\n", "Precedence: auto_reply\r\n", "List-Id: Dev <dev.example.org>\r\n"} {
conn, err := net.Dial("tcp", st.addr)
if err != nil {
t.Fatal(err)
}
c := smtp.NewClientLMTP(conn)
c.Hello("localhost")
c.Mail("alice@example.com", nil)
c.Rcpt("dev@example.org", nil)
w, _ := c.Data()
w.Write([]byte("From: Alice <alice@example.com>\r\n" + h + "Subject: x\r\n\r\nbody\r\n"))
if _, err := w.CloseWithLMTPResponse(); err != nil {
t.Fatalf("%q: a discarded message is acknowledged so the sender gets no bounce: %v", h, err)
}
c.Close()
}
if st.queued(t) != 0 {
t.Fatal("nothing may be queued")
}
}
func TestAutoGeneratedPostsAreDelivered(t *testing.T) {
st := newStack(t, nil)
for _, h := range []string{"Auto-Submitted: auto-generated\r\n", "Auto-Submitted: auto-notified\r\n"} {
conn, err := net.Dial("tcp", st.addr)
if err != nil {
t.Fatal(err)
}
c := smtp.NewClientLMTP(conn)
c.Hello("localhost")
c.Mail("alice@example.com", nil)
c.Rcpt("dev@example.org", nil)
w, _ := c.Data()
w.Write([]byte("From: Alice <alice@example.com>\r\n" + h + "Subject: nightly report\r\nMessage-ID: <" + strings.TrimSpace(h) + "@x>\r\n\r\nbody\r\n"))
if _, err := w.CloseWithLMTPResponse(); err != nil {
t.Fatalf("%q: %v", h, err)
}
c.Close()
}
if st.queued(t) == 0 {
t.Fatal("a cron or CI post from a member must be queued for delivery")
}
}
func TestAFullModerationQueueAnswers452(t *testing.T) {
st := newStack(t, nil)
l, _ := st.store.FindList("dev@example.org")
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
for i := 0; i < 500; i++ {
st.store.HoldMessage(l.ID, "x@y.com", []byte("Subject: s\r\n\r\nb"), "r")
}
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
var se *smtp.SMTPError
if !errors.As(res["dev@example.org"], &se) || se.Code != 452 {
t.Fatalf("a temporary refusal, so a legitimate sender retries later and a spam run goes away: %v", res["dev@example.org"])
}
}
func TestAMessageTooBigToHoldAnswers552(t *testing.T) {
st := newStack(t, nil)
l, _ := st.store.FindList("dev@example.org")
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
conn, err := net.Dial("tcp", st.addr)
if err != nil {
t.Fatal(err)
}
c := smtp.NewClientLMTP(conn)
defer c.Close()
c.Hello("localhost")
c.Mail("alice@example.com", nil)
c.Rcpt("dev@example.org", nil)
w, _ := c.Data()
w.Write([]byte("From: Alice <alice@example.com>\r\nSubject: huge\r\n\r\n"))
line := []byte(strings.Repeat("x", 76) + "\r\n")
for i := 0; i < (10<<20)/len(line)+10; i++ {
w.Write(line)
}
_, err = w.CloseWithLMTPResponse()
var de smtp.LMTPDataError
var se *smtp.SMTPError
if !errors.As(err, &de) || !errors.As(de["dev@example.org"], &se) || se.Code != 552 {
t.Fatalf("a permanent refusal: the sender is told to send it elsewhere: %v", err)
}
}
func TestTheSamePostDeliveredAgainIsAcceptedButNotQueuedTwice(t *testing.T) {
st := newStack(t, down{}) // the deliveries stay in the queue, where they can be counted
for i := 0; i < 2; i++ {
if res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}); res["dev@example.org"] != nil {
t.Fatalf("delivery %d: a duplicate is acknowledged with 250, got %v", i+1, res)
}
}
if n := st.queued(t); n != 2 {
t.Fatalf("queued = %d, want one delivery per member", n)
}
}