feat: end-to-end test and docs for the archive import
- Import a Mailman-style mbox through the CLI inside the container: encoded headers, multipart, escaped From lines, threading, idempotence - Check accent-insensitive search and the public page hiding addresses - Document the migration from Mailman in the README and the endpoint in OpenAPI
This commit is contained in:
parent
a170204ecb
commit
b777acc154
3 changed files with 111 additions and 0 deletions
18
README.md
18
README.md
|
|
@ -61,6 +61,7 @@ parsimony member import dev@example.org subscribers.txt # "email[,name]" per l
|
|||
parsimony held ls dev@example.org
|
||||
parsimony held approve 12
|
||||
parsimony archive ls dev@example.org --q "deploy"
|
||||
parsimony archive import dev@example.org dev.mbox # an mbox file or "-" for stdin
|
||||
parsimony queue ls
|
||||
parsimony --json list ls # JSON for scripts
|
||||
parsimony help
|
||||
|
|
@ -71,6 +72,23 @@ Exit status: 0 success, 1 failure (including lines that failed in `member import
|
|||
`parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works.
|
||||
In the container: `docker exec parsimony /parsimony list ls`.
|
||||
|
||||
## Migrating from Mailman
|
||||
|
||||
Both Mailman 2 and Mailman 3 (HyperKitty) can export a list's history as an mbox file.
|
||||
|
||||
```
|
||||
parsimony list add dev@example.org --title "Developers"
|
||||
parsimony member import dev@example.org subscribers.txt # "email[,name]" per line
|
||||
parsimony archive import dev@example.org dev.mbox
|
||||
```
|
||||
|
||||
The import reads standard mbox (a message starts at a `From addr Tue Jan 2 15:04:05 2024` line, `>From ` escapes are
|
||||
restored), decodes encoded headers, charsets and multipart bodies, and stores the plain text. It is safe to repeat: a
|
||||
message already in the archive (same `Message-ID`) is skipped. Threads are rebuilt from `In-Reply-To` whatever the order
|
||||
of the messages. Messages that cannot be read are listed and counted, and the exit status is 1; the others are kept.
|
||||
It works whatever the list's archive policy is (the policy only governs live mail). The admin UI has the same import on
|
||||
the Archive tab. Search is accent-insensitive: `caffe` finds "Caffè".
|
||||
|
||||
## Docker
|
||||
|
||||
The image runs as an unprivileged user (uid 65532) and works with a read-only root filesystem.
|
||||
|
|
|
|||
|
|
@ -275,3 +275,32 @@ paths:
|
|||
responses:
|
||||
'204': {description: Discarded}
|
||||
'404': {$ref: '#/components/responses/Error'}
|
||||
/lists/{id}/archive/import:
|
||||
parameters:
|
||||
- {name: id, in: path, required: true, schema: {type: integer}}
|
||||
post:
|
||||
summary: Import an mbox file into the archive
|
||||
description: >
|
||||
The request body is the mbox file itself (any content type, `application/mbox` by convention), streamed.
|
||||
Idempotent: a message already archived (same Message-ID) is counted as a duplicate. Threads are rebuilt at the end.
|
||||
Works whatever the list's archive policy. Messages that cannot be read are reported in the summary and do not
|
||||
fail the request; a read error while receiving the body does (400), keeping the messages read so far.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/mbox:
|
||||
schema: {type: string, format: binary}
|
||||
responses:
|
||||
'200':
|
||||
description: Summary
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
imported: {type: integer}
|
||||
duplicates: {type: integer}
|
||||
failed: {type: integer}
|
||||
errors: {type: array, items: {type: string}, description: The first 20 failures}
|
||||
'400': {$ref: '#/components/responses/Error'}
|
||||
'404': {$ref: '#/components/responses/Error'}
|
||||
|
|
|
|||
|
|
@ -544,6 +544,70 @@ func TestWholePath(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("a Mailman-style mbox can be imported and browsed", func(t *testing.T) {
|
||||
mbox := "From MAILER-DAEMON Mon Jan 1 10:00:00 2024\n" +
|
||||
"From: =?UTF-8?Q?Jos=C3=A9_Garc=C3=ADa?= <jose@old-list.example>\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\nMessage-ID: <old-1@old-list.example>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=BB\n\n" +
|
||||
"--BB\nContent-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\nCiao a tutti, scrivetemi a jose@old-list.example\n>From the old archive: caff=C3=A8!\n--BB\nContent-Type: text/html\n\n<p>html copy</p>\n--BB--\n\n" +
|
||||
"From carol@members.test Tue Jan 2 09:00:00 2024\n" +
|
||||
"From: Carol <carol@members.test>\nSubject: Re: Caff\xc3\xa8\nMessage-ID: <old-0@old-list.example>\nIn-Reply-To: <old-1@old-list.example>\nDate: Tue, 02 Jan 2024 09:00:00 +0000\n\nsee you there\n"
|
||||
imp := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
|
||||
imp.Stdin = strings.NewReader(mbox)
|
||||
out, err := imp.CombinedOutput()
|
||||
if err != nil || !strings.Contains(string(out), "Imported 2 messages") {
|
||||
t.Fatalf("archive import: %v\n%s", err, out)
|
||||
}
|
||||
// Importing again adds nothing.
|
||||
again := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
|
||||
again.Stdin = strings.NewReader(mbox)
|
||||
out, err = again.CombinedOutput()
|
||||
if err != nil || !strings.Contains(string(out), "Imported 0 messages") || !strings.Contains(string(out), "2 already present") {
|
||||
t.Fatalf("second import: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
var page struct {
|
||||
Messages []struct {
|
||||
ID int64 `json:"id"`
|
||||
ThreadID int64 `json:"thread_id"`
|
||||
Subject string `json:"subject"`
|
||||
Sender string `json:"sender"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=nueve", "", &page)
|
||||
if len(page.Messages) != 1 || page.Messages[0].Subject != "Caffè a las nueve" || page.Messages[0].Sender != "José García" {
|
||||
t.Fatalf("encoded headers must be decoded: %+v", page.Messages)
|
||||
}
|
||||
root := page.Messages[0]
|
||||
var msg struct {
|
||||
Body string `json:"body"`
|
||||
}
|
||||
mustCall(t, 200, "GET", fmt.Sprintf("/api/v1/archive/%d", root.ID), "", &msg)
|
||||
if !strings.Contains(msg.Body, "caffè!") || !strings.Contains(msg.Body, "\nFrom the old archive") || strings.Contains(msg.Body, "html copy") {
|
||||
t.Fatalf("body: %q", msg.Body)
|
||||
}
|
||||
var all struct {
|
||||
Messages []struct {
|
||||
ThreadID int64 `json:"thread_id"`
|
||||
Subject string `json:"subject"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=caffe", "", &all)
|
||||
if len(all.Messages) != 2 || all.Messages[0].ThreadID != all.Messages[1].ThreadID {
|
||||
t.Fatalf("the reply must be in its parent's thread: %+v", all.Messages)
|
||||
}
|
||||
|
||||
// The list's archive is public: imported messages show up on the web, with addresses hidden.
|
||||
resp, err := http.Get(fmt.Sprintf("%s/archive/%s/%d", webBase, list, root.ID))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 200 || !strings.Contains(string(b), "José García") || !strings.Contains(string(b), "jose@…") ||
|
||||
strings.Contains(string(b), "jose@old-list.example") {
|
||||
t.Fatalf("public page: %d\n%.700s", resp.StatusCode, b)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the queue ends empty", func(t *testing.T) {
|
||||
waitQueueEmpty(t)
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in a new issue