feat(cli): archive import from a file or stdin
- Add parsimony archive import <list> <file|->, streaming with no overall timeout - Print imported, already present and failed counts; report failing messages and exit 1 - Support --json
This commit is contained in:
parent
cd596e1810
commit
cc47784fdb
3 changed files with 158 additions and 8 deletions
|
|
@ -45,6 +45,8 @@ Administration (talks to the REST API):
|
|||
archive ls <list> [--q TEXT] [--limit N] [--offset N]
|
||||
archive show <id>
|
||||
archive rm <id> --yes
|
||||
archive import <list> <file|-> add the messages of an mbox file (Mailman export);
|
||||
safe to repeat, threads are rebuilt
|
||||
|
||||
queue ls outbound queue
|
||||
queue rm <id> discard one queued delivery
|
||||
|
|
@ -90,7 +92,7 @@ func Run(args []string, io IO, getenv func(string) string) int {
|
|||
"list": {"ls": c.listLs, "add": c.listAdd, "show": c.listShow, "set": c.listSet, "rm": c.listRm},
|
||||
"member": {"ls": c.memberLs, "add": c.memberAdd, "rm": c.memberRm, "import": c.memberImport},
|
||||
"held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard},
|
||||
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm},
|
||||
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport},
|
||||
"queue": {"ls": c.queueLs, "rm": c.queueRm},
|
||||
}
|
||||
group, ok := commands[rest[0]]
|
||||
|
|
@ -145,7 +147,11 @@ func (c *cmd) client() (*client, error) {
|
|||
if _, err := url.Parse(base); err != nil {
|
||||
return nil, fmt.Errorf("invalid URL %q", base)
|
||||
}
|
||||
c.cl = &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 30 * time.Second}}
|
||||
c.cl = &client{
|
||||
base: strings.TrimRight(base, "/"), token: token,
|
||||
http: &http.Client{Timeout: 30 * time.Second},
|
||||
upload: &http.Client{}, // big files: no overall timeout
|
||||
}
|
||||
return c.cl, nil
|
||||
}
|
||||
|
||||
|
|
@ -176,7 +182,8 @@ func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
|
|||
|
||||
type client struct {
|
||||
base, token string
|
||||
http *http.Client
|
||||
http *http.Client // ordinary requests, with a timeout
|
||||
upload *http.Client // streaming uploads, without one
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
|
|
@ -186,25 +193,35 @@ type apiError struct {
|
|||
|
||||
func (e *apiError) Error() string { return e.message }
|
||||
|
||||
// do sends a request and returns the raw response body (nil for 204).
|
||||
// do sends a JSON request and returns the raw response body (nil for 204).
|
||||
func (cl *client) do(method, path string, body any) (json.RawMessage, error) {
|
||||
var rd io.Reader
|
||||
ctype := ""
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rd = bytes.NewReader(b)
|
||||
rd, ctype = bytes.NewReader(b), "application/json"
|
||||
}
|
||||
return cl.send(cl.http, method, path, rd, ctype)
|
||||
}
|
||||
|
||||
// uploadTo streams r as the body of a POST, with no overall timeout.
|
||||
func (cl *client) uploadTo(path string, r io.Reader, contentType string) (json.RawMessage, error) {
|
||||
return cl.send(cl.upload, "POST", path, r, contentType)
|
||||
}
|
||||
|
||||
func (cl *client) send(hc *http.Client, method, path string, rd io.Reader, contentType string) (json.RawMessage, error) {
|
||||
req, err := http.NewRequest(method, cl.base+"/api/v1"+path, rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+cl.token)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
resp, err := cl.http.Do(req)
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach %s: %w", cl.base, errors.Unwrap(err))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -438,3 +438,77 @@ func TestQueue(t *testing.T) {
|
|||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
const cliMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice <alice@example.com>\nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" +
|
||||
"From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob <bob@example.com>\nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n"
|
||||
|
||||
func TestArchiveImportFromFileAndStdin(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
path := t.TempDir() + "/dev.mbox"
|
||||
if err := os.WriteFile(path, []byte(cliMbox), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := e.ok(t, "archive", "import", "dev@example.org", path)
|
||||
if !strings.Contains(out, "Imported 2 messages into dev@example.org") || strings.Contains(out, "failed") || strings.Contains(out, "already") {
|
||||
t.Fatalf("import: %q", out)
|
||||
}
|
||||
l, _ := e.s.FindList("dev@example.org")
|
||||
if _, total, _ := e.s.ListMessages(l.ID, 10, 0); total != 2 {
|
||||
t.Fatalf("stored %d", total)
|
||||
}
|
||||
|
||||
// Again, through stdin: nothing new, and it says so.
|
||||
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "archive", "import", "1", "-")
|
||||
if code != 0 || !strings.Contains(out, "Imported 0 messages") || !strings.Contains(out, "2 already present") {
|
||||
t.Fatalf("second import: %d %q %q", code, out, errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportReportsFailures(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
bad := cliMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n"
|
||||
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, bad, "archive", "import", "dev@example.org", "-")
|
||||
if code != 1 {
|
||||
t.Fatalf("a failed message must give a non-zero exit status, got %d", code)
|
||||
}
|
||||
if !strings.Contains(out, "Imported 2 messages") || !strings.Contains(out, "1 failed") {
|
||||
t.Fatalf("summary: %q", out)
|
||||
}
|
||||
if !strings.Contains(errs, "message 3") || !strings.Contains(errs, "no headers") {
|
||||
t.Fatalf("the failure is reported: %q", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportErrorsAndJSON(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
e.fails(t, 1, "no such file", "archive", "import", "dev@example.org", "/nonexistent/x.mbox")
|
||||
e.fails(t, 1, "no such list", "archive", "import", "ghost@example.org", "/dev/null")
|
||||
e.fails(t, 2, "usage: parsimony archive import", "archive", "import", "dev@example.org")
|
||||
_, out, _ := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "--json", "archive", "import", "dev@example.org", "-")
|
||||
var sum map[string]any
|
||||
if err := json.Unmarshal([]byte(out), &sum); err != nil || sum["imported"].(float64) != 2 {
|
||||
t.Fatalf("json: %v %q", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongRunningUploadsAreNotCutByTheShortRequestTimeout(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
var cl *client
|
||||
c := &cmd{getenv: func(k string) string {
|
||||
return map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}[k]
|
||||
}}
|
||||
cl, err := c.client()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cl.http.Timeout == 0 {
|
||||
t.Fatal("ordinary requests need a timeout")
|
||||
}
|
||||
if cl.upload == nil || cl.upload.Timeout != 0 {
|
||||
t.Fatal("uploads of big archives must not have an overall timeout")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -546,6 +546,65 @@ func (c *cmd) archiveRm(args []string) int {
|
|||
}, "Deleted archived message %d")
|
||||
}
|
||||
|
||||
func (c *cmd) archiveImport(args []string) int {
|
||||
pos, err := parseArgs(newFlagSet("archive import"), args)
|
||||
if err != nil {
|
||||
return c.usageError("%v", err)
|
||||
}
|
||||
if len(pos) != 2 {
|
||||
return c.usageError("usage: parsimony archive import <list> <file|->")
|
||||
}
|
||||
l, err := c.resolveList(pos[0])
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
var in io.Reader = c.io.In
|
||||
if pos[1] != "-" {
|
||||
f, err := os.Open(pos[1])
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
defer f.Close()
|
||||
in = f
|
||||
}
|
||||
cl, _ := c.client()
|
||||
raw, err := cl.uploadTo(fmt.Sprintf("/lists/%d/archive/import", l.ID), in, "application/mbox")
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
var sum struct {
|
||||
Imported int `json:"imported"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Failed int `json:"failed"`
|
||||
Errors []string `json:"errors"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &sum); err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
if c.json {
|
||||
c.printJSON(raw)
|
||||
} else {
|
||||
msg := fmt.Sprintf("Imported %s into %s", plural(sum.Imported, "message", "messages"), l.Address)
|
||||
if sum.Duplicates > 0 {
|
||||
msg += fmt.Sprintf(", %d already present", sum.Duplicates)
|
||||
}
|
||||
if sum.Failed > 0 {
|
||||
msg += fmt.Sprintf(", %d failed", sum.Failed)
|
||||
}
|
||||
fmt.Fprintln(c.io.Out, msg)
|
||||
}
|
||||
for _, e := range sum.Errors {
|
||||
fmt.Fprintln(c.io.Err, e)
|
||||
}
|
||||
if sum.Failed > len(sum.Errors) {
|
||||
fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors))
|
||||
}
|
||||
if sum.Failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// --- queue --------------------------------------------------------------------------
|
||||
|
||||
func (c *cmd) queueLs(args []string) int {
|
||||
|
|
|
|||
Loading…
Reference in a new issue