feat: end-to-end test and docs for the archive import

- Import a Mailman-style mbox through the CLI inside the container: encoded headers, multipart, escaped From lines, threading, idempotence
- Check accent-insensitive search and the public page hiding addresses
- Document the migration from Mailman in the README and the endpoint in OpenAPI
This commit is contained in:
tgiachi 2026-11-09 19:52:14 +01:00
commit b777acc154
3 changed files with 111 additions and 0 deletions

View file

@ -61,6 +61,7 @@ parsimony member import dev@example.org subscribers.txt # "email[,name]" per l
parsimony held ls dev@example.org
parsimony held approve 12
parsimony archive ls dev@example.org --q "deploy"
parsimony archive import dev@example.org dev.mbox # an mbox file or "-" for stdin
parsimony queue ls
parsimony --json list ls # JSON for scripts
parsimony help
@ -71,6 +72,23 @@ Exit status: 0 success, 1 failure (including lines that failed in `member import
`parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works.
In the container: `docker exec parsimony /parsimony list ls`.
## Migrating from Mailman
Both Mailman 2 and Mailman 3 (HyperKitty) can export a list's history as an mbox file.
```
parsimony list add dev@example.org --title "Developers"
parsimony member import dev@example.org subscribers.txt # "email[,name]" per line
parsimony archive import dev@example.org dev.mbox
```
The import reads standard mbox (a message starts at a `From addr Tue Jan 2 15:04:05 2024` line, `>From ` escapes are
restored), decodes encoded headers, charsets and multipart bodies, and stores the plain text. It is safe to repeat: a
message already in the archive (same `Message-ID`) is skipped. Threads are rebuilt from `In-Reply-To` whatever the order
of the messages. Messages that cannot be read are listed and counted, and the exit status is 1; the others are kept.
It works whatever the list's archive policy is (the policy only governs live mail). The admin UI has the same import on
the Archive tab. Search is accent-insensitive: `caffe` finds "Caffè".
## Docker
The image runs as an unprivileged user (uid 65532) and works with a read-only root filesystem.

View file

@ -275,3 +275,32 @@ paths:
responses:
'204': {description: Discarded}
'404': {$ref: '#/components/responses/Error'}
/lists/{id}/archive/import:
parameters:
- {name: id, in: path, required: true, schema: {type: integer}}
post:
summary: Import an mbox file into the archive
description: >
The request body is the mbox file itself (any content type, `application/mbox` by convention), streamed.
Idempotent: a message already archived (same Message-ID) is counted as a duplicate. Threads are rebuilt at the end.
Works whatever the list's archive policy. Messages that cannot be read are reported in the summary and do not
fail the request; a read error while receiving the body does (400), keeping the messages read so far.
requestBody:
required: true
content:
application/mbox:
schema: {type: string, format: binary}
responses:
'200':
description: Summary
content:
application/json:
schema:
type: object
properties:
imported: {type: integer}
duplicates: {type: integer}
failed: {type: integer}
errors: {type: array, items: {type: string}, description: The first 20 failures}
'400': {$ref: '#/components/responses/Error'}
'404': {$ref: '#/components/responses/Error'}

View file

@ -544,6 +544,70 @@ func TestWholePath(t *testing.T) {
}
})
t.Run("a Mailman-style mbox can be imported and browsed", func(t *testing.T) {
mbox := "From MAILER-DAEMON Mon Jan 1 10:00:00 2024\n" +
"From: =?UTF-8?Q?Jos=C3=A9_Garc=C3=ADa?= <jose@old-list.example>\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\nMessage-ID: <old-1@old-list.example>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=BB\n\n" +
"--BB\nContent-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\nCiao a tutti, scrivetemi a jose@old-list.example\n>From the old archive: caff=C3=A8!\n--BB\nContent-Type: text/html\n\n<p>html copy</p>\n--BB--\n\n" +
"From carol@members.test Tue Jan 2 09:00:00 2024\n" +
"From: Carol <carol@members.test>\nSubject: Re: Caff\xc3\xa8\nMessage-ID: <old-0@old-list.example>\nIn-Reply-To: <old-1@old-list.example>\nDate: Tue, 02 Jan 2024 09:00:00 +0000\n\nsee you there\n"
imp := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
imp.Stdin = strings.NewReader(mbox)
out, err := imp.CombinedOutput()
if err != nil || !strings.Contains(string(out), "Imported 2 messages") {
t.Fatalf("archive import: %v\n%s", err, out)
}
// Importing again adds nothing.
again := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
again.Stdin = strings.NewReader(mbox)
out, err = again.CombinedOutput()
if err != nil || !strings.Contains(string(out), "Imported 0 messages") || !strings.Contains(string(out), "2 already present") {
t.Fatalf("second import: %v\n%s", err, out)
}
var page struct {
Messages []struct {
ID int64 `json:"id"`
ThreadID int64 `json:"thread_id"`
Subject string `json:"subject"`
Sender string `json:"sender"`
} `json:"messages"`
}
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=nueve", "", &page)
if len(page.Messages) != 1 || page.Messages[0].Subject != "Caffè a las nueve" || page.Messages[0].Sender != "José García" {
t.Fatalf("encoded headers must be decoded: %+v", page.Messages)
}
root := page.Messages[0]
var msg struct {
Body string `json:"body"`
}
mustCall(t, 200, "GET", fmt.Sprintf("/api/v1/archive/%d", root.ID), "", &msg)
if !strings.Contains(msg.Body, "caffè!") || !strings.Contains(msg.Body, "\nFrom the old archive") || strings.Contains(msg.Body, "html copy") {
t.Fatalf("body: %q", msg.Body)
}
var all struct {
Messages []struct {
ThreadID int64 `json:"thread_id"`
Subject string `json:"subject"`
} `json:"messages"`
}
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=caffe", "", &all)
if len(all.Messages) != 2 || all.Messages[0].ThreadID != all.Messages[1].ThreadID {
t.Fatalf("the reply must be in its parent's thread: %+v", all.Messages)
}
// The list's archive is public: imported messages show up on the web, with addresses hidden.
resp, err := http.Get(fmt.Sprintf("%s/archive/%s/%d", webBase, list, root.ID))
if err != nil {
t.Fatal(err)
}
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != 200 || !strings.Contains(string(b), "José García") || !strings.Contains(string(b), "jose@…") ||
strings.Contains(string(b), "jose@old-list.example") {
t.Fatalf("public page: %d\n%.700s", resp.StatusCode, b)
}
})
t.Run("the queue ends empty", func(t *testing.T) {
waitQueueEmpty(t)
})