From cc47784fdbe2035704e20f353dfa0c4d46e7bc16 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 17:15:43 +0100 Subject: [PATCH] feat(cli): archive import from a file or stdin - Add parsimony archive import , streaming with no overall timeout - Print imported, already present and failed counts; report failing messages and exit 1 - Support --json --- internal/cli/cli.go | 33 +++++++++++++----- internal/cli/cli_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ internal/cli/commands.go | 59 ++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 8 deletions(-) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index f279966..2dfa020 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -45,6 +45,8 @@ Administration (talks to the REST API): archive ls [--q TEXT] [--limit N] [--offset N] archive show archive rm --yes + archive import add the messages of an mbox file (Mailman export); + safe to repeat, threads are rebuilt queue ls outbound queue queue rm discard one queued delivery @@ -90,7 +92,7 @@ func Run(args []string, io IO, getenv func(string) string) int { "list": {"ls": c.listLs, "add": c.listAdd, "show": c.listShow, "set": c.listSet, "rm": c.listRm}, "member": {"ls": c.memberLs, "add": c.memberAdd, "rm": c.memberRm, "import": c.memberImport}, "held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard}, - "archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm}, + "archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport}, "queue": {"ls": c.queueLs, "rm": c.queueRm}, } group, ok := commands[rest[0]] @@ -145,7 +147,11 @@ func (c *cmd) client() (*client, error) { if _, err := url.Parse(base); err != nil { return nil, fmt.Errorf("invalid URL %q", base) } - c.cl = &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 30 * time.Second}} + c.cl = &client{ + base: strings.TrimRight(base, "/"), token: token, + http: &http.Client{Timeout: 30 * time.Second}, + upload: &http.Client{}, // big files: no overall timeout + } return c.cl, nil } @@ -176,7 +182,8 @@ func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) { type client struct { base, token string - http *http.Client + http *http.Client // ordinary requests, with a timeout + upload *http.Client // streaming uploads, without one } type apiError struct { @@ -186,25 +193,35 @@ type apiError struct { func (e *apiError) Error() string { return e.message } -// do sends a request and returns the raw response body (nil for 204). +// do sends a JSON request and returns the raw response body (nil for 204). func (cl *client) do(method, path string, body any) (json.RawMessage, error) { var rd io.Reader + ctype := "" if body != nil { b, err := json.Marshal(body) if err != nil { return nil, err } - rd = bytes.NewReader(b) + rd, ctype = bytes.NewReader(b), "application/json" } + return cl.send(cl.http, method, path, rd, ctype) +} + +// uploadTo streams r as the body of a POST, with no overall timeout. +func (cl *client) uploadTo(path string, r io.Reader, contentType string) (json.RawMessage, error) { + return cl.send(cl.upload, "POST", path, r, contentType) +} + +func (cl *client) send(hc *http.Client, method, path string, rd io.Reader, contentType string) (json.RawMessage, error) { req, err := http.NewRequest(method, cl.base+"/api/v1"+path, rd) if err != nil { return nil, err } req.Header.Set("Authorization", "Bearer "+cl.token) - if body != nil { - req.Header.Set("Content-Type", "application/json") + if contentType != "" { + req.Header.Set("Content-Type", contentType) } - resp, err := cl.http.Do(req) + resp, err := hc.Do(req) if err != nil { return nil, fmt.Errorf("cannot reach %s: %w", cl.base, errors.Unwrap(err)) } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 34e8c7c..2fb14ee 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -438,3 +438,77 @@ func TestQueue(t *testing.T) { t.Fatalf("pending = %d", st.Pending) } } + +const cliMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice \nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" + + "From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob \nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n" + +func TestArchiveImportFromFileAndStdin(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + path := t.TempDir() + "/dev.mbox" + if err := os.WriteFile(path, []byte(cliMbox), 0o600); err != nil { + t.Fatal(err) + } + out := e.ok(t, "archive", "import", "dev@example.org", path) + if !strings.Contains(out, "Imported 2 messages into dev@example.org") || strings.Contains(out, "failed") || strings.Contains(out, "already") { + t.Fatalf("import: %q", out) + } + l, _ := e.s.FindList("dev@example.org") + if _, total, _ := e.s.ListMessages(l.ID, 10, 0); total != 2 { + t.Fatalf("stored %d", total) + } + + // Again, through stdin: nothing new, and it says so. + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "archive", "import", "1", "-") + if code != 0 || !strings.Contains(out, "Imported 0 messages") || !strings.Contains(out, "2 already present") { + t.Fatalf("second import: %d %q %q", code, out, errs) + } +} + +func TestArchiveImportReportsFailures(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + bad := cliMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n" + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, bad, "archive", "import", "dev@example.org", "-") + if code != 1 { + t.Fatalf("a failed message must give a non-zero exit status, got %d", code) + } + if !strings.Contains(out, "Imported 2 messages") || !strings.Contains(out, "1 failed") { + t.Fatalf("summary: %q", out) + } + if !strings.Contains(errs, "message 3") || !strings.Contains(errs, "no headers") { + t.Fatalf("the failure is reported: %q", errs) + } +} + +func TestArchiveImportErrorsAndJSON(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + e.fails(t, 1, "no such file", "archive", "import", "dev@example.org", "/nonexistent/x.mbox") + e.fails(t, 1, "no such list", "archive", "import", "ghost@example.org", "/dev/null") + e.fails(t, 2, "usage: parsimony archive import", "archive", "import", "dev@example.org") + _, out, _ := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "--json", "archive", "import", "dev@example.org", "-") + var sum map[string]any + if err := json.Unmarshal([]byte(out), &sum); err != nil || sum["imported"].(float64) != 2 { + t.Fatalf("json: %v %q", err, out) + } +} + +func TestLongRunningUploadsAreNotCutByTheShortRequestTimeout(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + var cl *client + c := &cmd{getenv: func(k string) string { + return map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}[k] + }} + cl, err := c.client() + if err != nil { + t.Fatal(err) + } + if cl.http.Timeout == 0 { + t.Fatal("ordinary requests need a timeout") + } + if cl.upload == nil || cl.upload.Timeout != 0 { + t.Fatal("uploads of big archives must not have an overall timeout") + } +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go index cf92e96..2da2c15 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -546,6 +546,65 @@ func (c *cmd) archiveRm(args []string) int { }, "Deleted archived message %d") } +func (c *cmd) archiveImport(args []string) int { + pos, err := parseArgs(newFlagSet("archive import"), args) + if err != nil { + return c.usageError("%v", err) + } + if len(pos) != 2 { + return c.usageError("usage: parsimony archive import ") + } + l, err := c.resolveList(pos[0]) + if err != nil { + return c.fail(err) + } + var in io.Reader = c.io.In + if pos[1] != "-" { + f, err := os.Open(pos[1]) + if err != nil { + return c.fail(err) + } + defer f.Close() + in = f + } + cl, _ := c.client() + raw, err := cl.uploadTo(fmt.Sprintf("/lists/%d/archive/import", l.ID), in, "application/mbox") + if err != nil { + return c.fail(err) + } + var sum struct { + Imported int `json:"imported"` + Duplicates int `json:"duplicates"` + Failed int `json:"failed"` + Errors []string `json:"errors"` + } + if err := json.Unmarshal(raw, &sum); err != nil { + return c.fail(err) + } + if c.json { + c.printJSON(raw) + } else { + msg := fmt.Sprintf("Imported %s into %s", plural(sum.Imported, "message", "messages"), l.Address) + if sum.Duplicates > 0 { + msg += fmt.Sprintf(", %d already present", sum.Duplicates) + } + if sum.Failed > 0 { + msg += fmt.Sprintf(", %d failed", sum.Failed) + } + fmt.Fprintln(c.io.Out, msg) + } + for _, e := range sum.Errors { + fmt.Fprintln(c.io.Err, e) + } + if sum.Failed > len(sum.Errors) { + fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors)) + } + if sum.Failed > 0 { + return 1 + } + return 0 +} + // --- queue -------------------------------------------------------------------------- func (c *cmd) queueLs(args []string) int {