From b777acc1547d596df2552f80dfac0be03876e746 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 19:52:14 +0100 Subject: [PATCH] feat: end-to-end test and docs for the archive import - Import a Mailman-style mbox through the CLI inside the container: encoded headers, multipart, escaped From lines, threading, idempotence - Check accent-insensitive search and the public page hiding addresses - Document the migration from Mailman in the README and the endpoint in OpenAPI --- README.md | 18 +++++++++++++ docs/openapi.yaml | 29 +++++++++++++++++++++ e2e/e2e_test.go | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) diff --git a/README.md b/README.md index df253b3..97ae3e0 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ parsimony member import dev@example.org subscribers.txt # "email[,name]" per l parsimony held ls dev@example.org parsimony held approve 12 parsimony archive ls dev@example.org --q "deploy" +parsimony archive import dev@example.org dev.mbox # an mbox file or "-" for stdin parsimony queue ls parsimony --json list ls # JSON for scripts parsimony help @@ -71,6 +72,23 @@ Exit status: 0 success, 1 failure (including lines that failed in `member import `parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works. In the container: `docker exec parsimony /parsimony list ls`. +## Migrating from Mailman + +Both Mailman 2 and Mailman 3 (HyperKitty) can export a list's history as an mbox file. + +``` +parsimony list add dev@example.org --title "Developers" +parsimony member import dev@example.org subscribers.txt # "email[,name]" per line +parsimony archive import dev@example.org dev.mbox +``` + +The import reads standard mbox (a message starts at a `From addr Tue Jan 2 15:04:05 2024` line, `>From ` escapes are +restored), decodes encoded headers, charsets and multipart bodies, and stores the plain text. It is safe to repeat: a +message already in the archive (same `Message-ID`) is skipped. Threads are rebuilt from `In-Reply-To` whatever the order +of the messages. Messages that cannot be read are listed and counted, and the exit status is 1; the others are kept. +It works whatever the list's archive policy is (the policy only governs live mail). The admin UI has the same import on +the Archive tab. Search is accent-insensitive: `caffe` finds "Caffè". + ## Docker The image runs as an unprivileged user (uid 65532) and works with a read-only root filesystem. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 52395a0..9b6a132 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -275,3 +275,32 @@ paths: responses: '204': {description: Discarded} '404': {$ref: '#/components/responses/Error'} + /lists/{id}/archive/import: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + post: + summary: Import an mbox file into the archive + description: > + The request body is the mbox file itself (any content type, `application/mbox` by convention), streamed. + Idempotent: a message already archived (same Message-ID) is counted as a duplicate. Threads are rebuilt at the end. + Works whatever the list's archive policy. Messages that cannot be read are reported in the summary and do not + fail the request; a read error while receiving the body does (400), keeping the messages read so far. + requestBody: + required: true + content: + application/mbox: + schema: {type: string, format: binary} + responses: + '200': + description: Summary + content: + application/json: + schema: + type: object + properties: + imported: {type: integer} + duplicates: {type: integer} + failed: {type: integer} + errors: {type: array, items: {type: string}, description: The first 20 failures} + '400': {$ref: '#/components/responses/Error'} + '404': {$ref: '#/components/responses/Error'} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 91d2a42..714d7eb 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -544,6 +544,70 @@ func TestWholePath(t *testing.T) { } }) + t.Run("a Mailman-style mbox can be imported and browsed", func(t *testing.T) { + mbox := "From MAILER-DAEMON Mon Jan 1 10:00:00 2024\n" + + "From: =?UTF-8?Q?Jos=C3=A9_Garc=C3=ADa?= \nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\nMessage-ID: \nDate: Mon, 01 Jan 2024 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=BB\n\n" + + "--BB\nContent-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\nCiao a tutti, scrivetemi a jose@old-list.example\n>From the old archive: caff=C3=A8!\n--BB\nContent-Type: text/html\n\n

html copy

\n--BB--\n\n" + + "From carol@members.test Tue Jan 2 09:00:00 2024\n" + + "From: Carol \nSubject: Re: Caff\xc3\xa8\nMessage-ID: \nIn-Reply-To: \nDate: Tue, 02 Jan 2024 09:00:00 +0000\n\nsee you there\n" + imp := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-") + imp.Stdin = strings.NewReader(mbox) + out, err := imp.CombinedOutput() + if err != nil || !strings.Contains(string(out), "Imported 2 messages") { + t.Fatalf("archive import: %v\n%s", err, out) + } + // Importing again adds nothing. + again := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-") + again.Stdin = strings.NewReader(mbox) + out, err = again.CombinedOutput() + if err != nil || !strings.Contains(string(out), "Imported 0 messages") || !strings.Contains(string(out), "2 already present") { + t.Fatalf("second import: %v\n%s", err, out) + } + + var page struct { + Messages []struct { + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + Subject string `json:"subject"` + Sender string `json:"sender"` + } `json:"messages"` + } + mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=nueve", "", &page) + if len(page.Messages) != 1 || page.Messages[0].Subject != "Caffè a las nueve" || page.Messages[0].Sender != "José García" { + t.Fatalf("encoded headers must be decoded: %+v", page.Messages) + } + root := page.Messages[0] + var msg struct { + Body string `json:"body"` + } + mustCall(t, 200, "GET", fmt.Sprintf("/api/v1/archive/%d", root.ID), "", &msg) + if !strings.Contains(msg.Body, "caffè!") || !strings.Contains(msg.Body, "\nFrom the old archive") || strings.Contains(msg.Body, "html copy") { + t.Fatalf("body: %q", msg.Body) + } + var all struct { + Messages []struct { + ThreadID int64 `json:"thread_id"` + Subject string `json:"subject"` + } `json:"messages"` + } + mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=caffe", "", &all) + if len(all.Messages) != 2 || all.Messages[0].ThreadID != all.Messages[1].ThreadID { + t.Fatalf("the reply must be in its parent's thread: %+v", all.Messages) + } + + // The list's archive is public: imported messages show up on the web, with addresses hidden. + resp, err := http.Get(fmt.Sprintf("%s/archive/%s/%d", webBase, list, root.ID)) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || !strings.Contains(string(b), "José García") || !strings.Contains(string(b), "jose@…") || + strings.Contains(string(b), "jose@old-list.example") { + t.Fatalf("public page: %d\n%.700s", resp.StatusCode, b) + } + }) + t.Run("the queue ends empty", func(t *testing.T) { waitQueueEmpty(t) })