feat(cli): archive import from a file or stdin

- Add parsimony archive import <list> <file|->, streaming with no overall timeout
- Print imported, already present and failed counts; report failing messages and exit 1
- Support --json
This commit is contained in:
tgiachi 2026-11-09 17:15:43 +01:00
commit cc47784fdb
3 changed files with 158 additions and 8 deletions

View file

@ -45,6 +45,8 @@ Administration (talks to the REST API):
archive ls <list> [--q TEXT] [--limit N] [--offset N]
archive show <id>
archive rm <id> --yes
archive import <list> <file|-> add the messages of an mbox file (Mailman export);
safe to repeat, threads are rebuilt
queue ls outbound queue
queue rm <id> discard one queued delivery
@ -90,7 +92,7 @@ func Run(args []string, io IO, getenv func(string) string) int {
"list": {"ls": c.listLs, "add": c.listAdd, "show": c.listShow, "set": c.listSet, "rm": c.listRm},
"member": {"ls": c.memberLs, "add": c.memberAdd, "rm": c.memberRm, "import": c.memberImport},
"held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard},
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm},
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport},
"queue": {"ls": c.queueLs, "rm": c.queueRm},
}
group, ok := commands[rest[0]]
@ -145,7 +147,11 @@ func (c *cmd) client() (*client, error) {
if _, err := url.Parse(base); err != nil {
return nil, fmt.Errorf("invalid URL %q", base)
}
c.cl = &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 30 * time.Second}}
c.cl = &client{
base: strings.TrimRight(base, "/"), token: token,
http: &http.Client{Timeout: 30 * time.Second},
upload: &http.Client{}, // big files: no overall timeout
}
return c.cl, nil
}
@ -176,7 +182,8 @@ func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
type client struct {
base, token string
http *http.Client
http *http.Client // ordinary requests, with a timeout
upload *http.Client // streaming uploads, without one
}
type apiError struct {
@ -186,25 +193,35 @@ type apiError struct {
func (e *apiError) Error() string { return e.message }
// do sends a request and returns the raw response body (nil for 204).
// do sends a JSON request and returns the raw response body (nil for 204).
func (cl *client) do(method, path string, body any) (json.RawMessage, error) {
var rd io.Reader
ctype := ""
if body != nil {
b, err := json.Marshal(body)
if err != nil {
return nil, err
}
rd = bytes.NewReader(b)
rd, ctype = bytes.NewReader(b), "application/json"
}
return cl.send(cl.http, method, path, rd, ctype)
}
// uploadTo streams r as the body of a POST, with no overall timeout.
func (cl *client) uploadTo(path string, r io.Reader, contentType string) (json.RawMessage, error) {
return cl.send(cl.upload, "POST", path, r, contentType)
}
func (cl *client) send(hc *http.Client, method, path string, rd io.Reader, contentType string) (json.RawMessage, error) {
req, err := http.NewRequest(method, cl.base+"/api/v1"+path, rd)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+cl.token)
if body != nil {
req.Header.Set("Content-Type", "application/json")
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
resp, err := cl.http.Do(req)
resp, err := hc.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot reach %s: %w", cl.base, errors.Unwrap(err))
}

View file

@ -438,3 +438,77 @@ func TestQueue(t *testing.T) {
t.Fatalf("pending = %d", st.Pending)
}
}
const cliMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice <alice@example.com>\nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" +
"From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob <bob@example.com>\nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n"
func TestArchiveImportFromFileAndStdin(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
path := t.TempDir() + "/dev.mbox"
if err := os.WriteFile(path, []byte(cliMbox), 0o600); err != nil {
t.Fatal(err)
}
out := e.ok(t, "archive", "import", "dev@example.org", path)
if !strings.Contains(out, "Imported 2 messages into dev@example.org") || strings.Contains(out, "failed") || strings.Contains(out, "already") {
t.Fatalf("import: %q", out)
}
l, _ := e.s.FindList("dev@example.org")
if _, total, _ := e.s.ListMessages(l.ID, 10, 0); total != 2 {
t.Fatalf("stored %d", total)
}
// Again, through stdin: nothing new, and it says so.
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "archive", "import", "1", "-")
if code != 0 || !strings.Contains(out, "Imported 0 messages") || !strings.Contains(out, "2 already present") {
t.Fatalf("second import: %d %q %q", code, out, errs)
}
}
func TestArchiveImportReportsFailures(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
bad := cliMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n"
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, bad, "archive", "import", "dev@example.org", "-")
if code != 1 {
t.Fatalf("a failed message must give a non-zero exit status, got %d", code)
}
if !strings.Contains(out, "Imported 2 messages") || !strings.Contains(out, "1 failed") {
t.Fatalf("summary: %q", out)
}
if !strings.Contains(errs, "message 3") || !strings.Contains(errs, "no headers") {
t.Fatalf("the failure is reported: %q", errs)
}
}
func TestArchiveImportErrorsAndJSON(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
e.fails(t, 1, "no such file", "archive", "import", "dev@example.org", "/nonexistent/x.mbox")
e.fails(t, 1, "no such list", "archive", "import", "ghost@example.org", "/dev/null")
e.fails(t, 2, "usage: parsimony archive import", "archive", "import", "dev@example.org")
_, out, _ := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "--json", "archive", "import", "dev@example.org", "-")
var sum map[string]any
if err := json.Unmarshal([]byte(out), &sum); err != nil || sum["imported"].(float64) != 2 {
t.Fatalf("json: %v %q", err, out)
}
}
func TestLongRunningUploadsAreNotCutByTheShortRequestTimeout(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
var cl *client
c := &cmd{getenv: func(k string) string {
return map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}[k]
}}
cl, err := c.client()
if err != nil {
t.Fatal(err)
}
if cl.http.Timeout == 0 {
t.Fatal("ordinary requests need a timeout")
}
if cl.upload == nil || cl.upload.Timeout != 0 {
t.Fatal("uploads of big archives must not have an overall timeout")
}
}

View file

@ -546,6 +546,65 @@ func (c *cmd) archiveRm(args []string) int {
}, "Deleted archived message %d")
}
func (c *cmd) archiveImport(args []string) int {
pos, err := parseArgs(newFlagSet("archive import"), args)
if err != nil {
return c.usageError("%v", err)
}
if len(pos) != 2 {
return c.usageError("usage: parsimony archive import <list> <file|->")
}
l, err := c.resolveList(pos[0])
if err != nil {
return c.fail(err)
}
var in io.Reader = c.io.In
if pos[1] != "-" {
f, err := os.Open(pos[1])
if err != nil {
return c.fail(err)
}
defer f.Close()
in = f
}
cl, _ := c.client()
raw, err := cl.uploadTo(fmt.Sprintf("/lists/%d/archive/import", l.ID), in, "application/mbox")
if err != nil {
return c.fail(err)
}
var sum struct {
Imported int `json:"imported"`
Duplicates int `json:"duplicates"`
Failed int `json:"failed"`
Errors []string `json:"errors"`
}
if err := json.Unmarshal(raw, &sum); err != nil {
return c.fail(err)
}
if c.json {
c.printJSON(raw)
} else {
msg := fmt.Sprintf("Imported %s into %s", plural(sum.Imported, "message", "messages"), l.Address)
if sum.Duplicates > 0 {
msg += fmt.Sprintf(", %d already present", sum.Duplicates)
}
if sum.Failed > 0 {
msg += fmt.Sprintf(", %d failed", sum.Failed)
}
fmt.Fprintln(c.io.Out, msg)
}
for _, e := range sum.Errors {
fmt.Fprintln(c.io.Err, e)
}
if sum.Failed > len(sum.Errors) {
fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors))
}
if sum.Failed > 0 {
return 1
}
return 0
}
// --- queue --------------------------------------------------------------------------
func (c *cmd) queueLs(args []string) int {