- Rename the Go module from codeberg.org/squidd/parsimony to git.kmalloc.org/squid/parsimony in go.mod and every import. - Fix .gitignore: the bare `parsimony` pattern also ignored the cmd/parsimony folder, so the main package was never committed and a clean checkout could not be built. Anchor it to the root (`/parsimony`) and add cmd/parsimony.
353 lines
10 KiB
Go
353 lines
10 KiB
Go
package ingest
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/emersion/go-smtp"
|
|
|
|
"git.kmalloc.org/squid/parsimony/internal/outbound"
|
|
"git.kmalloc.org/squid/parsimony/internal/queue"
|
|
"git.kmalloc.org/squid/parsimony/internal/router"
|
|
"git.kmalloc.org/squid/parsimony/internal/store"
|
|
)
|
|
|
|
const msg = "From: Alice <alice@example.com>\r\nSubject: hi\r\n\r\nbody\r\n"
|
|
|
|
// stack is the LMTP server with a real queue worker in front of a fake relay.
|
|
type stack struct {
|
|
addr string
|
|
relay *outbound.Fake
|
|
store *store.Store
|
|
}
|
|
|
|
func newStack(t *testing.T, sender outbound.Sender) *stack {
|
|
t.Helper()
|
|
s, err := store.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.AddMember(l.ID, "alice@example.com", "Alice")
|
|
s.AddMember(l.ID, "bob@example.com", "Bob")
|
|
|
|
relay := &outbound.Fake{}
|
|
if sender == nil {
|
|
sender = relay
|
|
}
|
|
rt := router.New(s)
|
|
w := &queue.Worker{Store: s, Sender: sender, Interval: 10 * time.Millisecond}
|
|
rt.OnEnqueue = w.Wake
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
go func() { w.Run(ctx); close(done) }()
|
|
t.Cleanup(func() { cancel(); <-done })
|
|
|
|
srv := NewServer(rt)
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Serve(ln)
|
|
t.Cleanup(func() { srv.Close() })
|
|
return &stack{addr: ln.Addr().String(), relay: relay, store: s}
|
|
}
|
|
|
|
// sent waits until the relay has received n messages and returns them.
|
|
func (st *stack) sent(t *testing.T, n int) []outbound.Sent {
|
|
t.Helper()
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
got := st.relay.Messages()
|
|
if len(got) >= n {
|
|
return got
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("relay got %d messages, want %d", len(got), n)
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// queued is how many deliveries still wait in the outbox.
|
|
func (st *stack) queued(t *testing.T) int {
|
|
t.Helper()
|
|
stats, err := st.store.OutboxStats()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stats.Pending
|
|
}
|
|
|
|
// deliver sends msg over LMTP and returns the per-recipient errors.
|
|
func deliver(t *testing.T, addr, from string, rcpts []string) map[string]error {
|
|
t.Helper()
|
|
conn, err := net.Dial("tcp", addr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := smtp.NewClientLMTP(conn)
|
|
defer c.Close()
|
|
if err := c.Hello("localhost"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Mail(from, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, r := range rcpts {
|
|
if err := c.Rcpt(r, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
w, err := c.Data()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.Write([]byte(msg))
|
|
resp, err := w.CloseWithLMTPResponse()
|
|
out := map[string]error{}
|
|
var de smtp.LMTPDataError
|
|
if errors.As(err, &de) {
|
|
for k, v := range de {
|
|
out[k] = v
|
|
}
|
|
} else if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for k := range resp {
|
|
out[k] = nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestDeliverFansOut(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
|
if res["dev@example.org"] != nil {
|
|
t.Fatalf("unexpected error: %v", res)
|
|
}
|
|
got := st.sent(t, 2)
|
|
if len(got) != 2 {
|
|
t.Fatalf("want one send per member, got %d", len(got))
|
|
}
|
|
m := got[0]
|
|
if len(m.To) != 1 {
|
|
t.Errorf("recipients = %v", m.To)
|
|
}
|
|
if !strings.HasPrefix(m.From, "dev+bounce-") {
|
|
t.Errorf("envelope from = %q", m.From)
|
|
}
|
|
if !strings.Contains(string(m.Data), "List-Id: Dev <dev.example.org>") {
|
|
t.Errorf("missing List-Id: %q", m.Data)
|
|
}
|
|
if !strings.Contains(string(m.Data), "List-Unsubscribe: <mailto:dev+leave@example.org>") {
|
|
t.Errorf("missing per-recipient unsubscribe header: %q", m.Data)
|
|
}
|
|
}
|
|
|
|
func TestNonMemberRejectedPermanently(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
res := deliver(t, st.addr, "mallory@evil.com", []string{"dev@example.org"})
|
|
var se *smtp.SMTPError
|
|
if !errors.As(res["dev@example.org"], &se) || se.Code != 550 {
|
|
t.Fatalf("want 550, got %v", res["dev@example.org"])
|
|
}
|
|
if st.queued(t) != 0 {
|
|
t.Fatal("nothing may be queued")
|
|
}
|
|
}
|
|
|
|
func TestUnknownListRejected(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
res := deliver(t, st.addr, "alice@example.com", []string{"nope@example.org"})
|
|
var se *smtp.SMTPError
|
|
if !errors.As(res["nope@example.org"], &se) || se.Code != 550 {
|
|
t.Fatalf("want 550, got %v", res["nope@example.org"])
|
|
}
|
|
}
|
|
|
|
func TestMixedRecipients(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"})
|
|
if res["dev@example.org"] != nil || res["nope@example.org"] == nil {
|
|
t.Fatalf("results: %v", res)
|
|
}
|
|
st.sent(t, 2)
|
|
}
|
|
|
|
// down is a relay that refuses everything.
|
|
type down struct{}
|
|
|
|
func (down) Send(context.Context, string, []string, []byte) error { return errors.New("relay down") }
|
|
|
|
func TestRelayDownStillAcceptsAndKeepsTheMailQueued(t *testing.T) {
|
|
st := newStack(t, down{})
|
|
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
|
if res["dev@example.org"] != nil {
|
|
t.Fatalf("a queued post is accepted even when the relay is down: %v", res)
|
|
}
|
|
// The worker tries and schedules retries; nothing is lost.
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
stats, _ := st.store.OutboxStats()
|
|
if stats.Pending == 2 && stats.Retrying == 2 {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("queue: %+v", stats)
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestQueueFailureIsTemporary(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
st.store.Close() // the database goes away under the server
|
|
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
|
var se *smtp.SMTPError
|
|
if !errors.As(res["dev@example.org"], &se) || se.Code != 451 {
|
|
t.Fatalf("want 451 so the MTA retries, got %v", res["dev@example.org"])
|
|
}
|
|
}
|
|
|
|
func TestJoinCommandOverLMTP(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
res := deliver(t, st.addr, "carol@x.com", []string{"dev+join@example.org"})
|
|
if res["dev+join@example.org"] != nil {
|
|
t.Fatalf("join: %v", res)
|
|
}
|
|
got := st.sent(t, 1)
|
|
if len(got[0].To) != 1 || got[0].To[0] != "carol@x.com" {
|
|
t.Fatalf("confirmation mail: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestBadTokenRejected(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
res := deliver(t, st.addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"})
|
|
var se *smtp.SMTPError
|
|
if !errors.As(res["dev+confirm-deadbeef@example.org"], &se) || se.Code != 550 {
|
|
t.Fatalf("want 550, got %v", res)
|
|
}
|
|
}
|
|
|
|
func TestModeratedPostIsAcceptedButNotQueued(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
l, _ := st.store.FindList("dev@example.org")
|
|
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
|
|
|
res := deliver(t, st.addr, "stranger@x.com", []string{"dev@example.org"})
|
|
if res["dev@example.org"] != nil {
|
|
t.Fatalf("held mail must be accepted: %v", res)
|
|
}
|
|
if st.queued(t) != 0 {
|
|
t.Fatal("nothing may be queued for delivery")
|
|
}
|
|
if h, _ := st.store.ListHeld(l.ID); len(h) != 1 {
|
|
t.Fatalf("held = %d", len(h))
|
|
}
|
|
}
|
|
|
|
func TestAutoReplyAndLoopsAreAcceptedButNotDelivered(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
for _, h := range []string{"Auto-Submitted: auto-replied\r\n", "Precedence: auto_reply\r\n", "List-Id: Dev <dev.example.org>\r\n"} {
|
|
conn, err := net.Dial("tcp", st.addr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := smtp.NewClientLMTP(conn)
|
|
c.Hello("localhost")
|
|
c.Mail("alice@example.com", nil)
|
|
c.Rcpt("dev@example.org", nil)
|
|
w, _ := c.Data()
|
|
w.Write([]byte("From: Alice <alice@example.com>\r\n" + h + "Subject: x\r\n\r\nbody\r\n"))
|
|
if _, err := w.CloseWithLMTPResponse(); err != nil {
|
|
t.Fatalf("%q: a discarded message is acknowledged so the sender gets no bounce: %v", h, err)
|
|
}
|
|
c.Close()
|
|
}
|
|
if st.queued(t) != 0 {
|
|
t.Fatal("nothing may be queued")
|
|
}
|
|
}
|
|
|
|
func TestAutoGeneratedPostsAreDelivered(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
for _, h := range []string{"Auto-Submitted: auto-generated\r\n", "Auto-Submitted: auto-notified\r\n"} {
|
|
conn, err := net.Dial("tcp", st.addr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := smtp.NewClientLMTP(conn)
|
|
c.Hello("localhost")
|
|
c.Mail("alice@example.com", nil)
|
|
c.Rcpt("dev@example.org", nil)
|
|
w, _ := c.Data()
|
|
w.Write([]byte("From: Alice <alice@example.com>\r\n" + h + "Subject: nightly report\r\nMessage-ID: <" + strings.TrimSpace(h) + "@x>\r\n\r\nbody\r\n"))
|
|
if _, err := w.CloseWithLMTPResponse(); err != nil {
|
|
t.Fatalf("%q: %v", h, err)
|
|
}
|
|
c.Close()
|
|
}
|
|
if st.queued(t) == 0 {
|
|
t.Fatal("a cron or CI post from a member must be queued for delivery")
|
|
}
|
|
}
|
|
|
|
func TestAFullModerationQueueAnswers452(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
l, _ := st.store.FindList("dev@example.org")
|
|
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
|
for i := 0; i < 500; i++ {
|
|
st.store.HoldMessage(l.ID, "x@y.com", []byte("Subject: s\r\n\r\nb"), "r")
|
|
}
|
|
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
|
var se *smtp.SMTPError
|
|
if !errors.As(res["dev@example.org"], &se) || se.Code != 452 {
|
|
t.Fatalf("a temporary refusal, so a legitimate sender retries later and a spam run goes away: %v", res["dev@example.org"])
|
|
}
|
|
}
|
|
|
|
func TestAMessageTooBigToHoldAnswers552(t *testing.T) {
|
|
st := newStack(t, nil)
|
|
l, _ := st.store.FindList("dev@example.org")
|
|
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
|
conn, err := net.Dial("tcp", st.addr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := smtp.NewClientLMTP(conn)
|
|
defer c.Close()
|
|
c.Hello("localhost")
|
|
c.Mail("alice@example.com", nil)
|
|
c.Rcpt("dev@example.org", nil)
|
|
w, _ := c.Data()
|
|
w.Write([]byte("From: Alice <alice@example.com>\r\nSubject: huge\r\n\r\n"))
|
|
line := []byte(strings.Repeat("x", 76) + "\r\n")
|
|
for i := 0; i < (10<<20)/len(line)+10; i++ {
|
|
w.Write(line)
|
|
}
|
|
_, err = w.CloseWithLMTPResponse()
|
|
var de smtp.LMTPDataError
|
|
var se *smtp.SMTPError
|
|
if !errors.As(err, &de) || !errors.As(de["dev@example.org"], &se) || se.Code != 552 {
|
|
t.Fatalf("a permanent refusal: the sender is told to send it elsewhere: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTheSamePostDeliveredAgainIsAcceptedButNotQueuedTwice(t *testing.T) {
|
|
st := newStack(t, down{}) // the deliveries stay in the queue, where they can be counted
|
|
for i := 0; i < 2; i++ {
|
|
if res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}); res["dev@example.org"] != nil {
|
|
t.Fatalf("delivery %d: a duplicate is acknowledged with 250, got %v", i+1, res)
|
|
}
|
|
}
|
|
if n := st.queued(t); n != 2 {
|
|
t.Fatalf("queued = %d, want one delivery per member", n)
|
|
}
|
|
}
|