sonda/internal/api/http/naming_test.go
tgiachi 1b79a7370d feat(naming): an admin section that shows what Claude costs
- Every question asked of Claude is recorded: when, the kind of list, the model, the tokens read and written (also for an answer that was no name but was billed) and how it ended: a name made, a name taken from the kept ones, or a failure and why. Nothing of the facts that were sent is recorded. Records older than a year are removed (migration 0022).
- GET /providers/anthropic/usage (admin) gives the totals of the last days (1 to 365, 30 by default), a row for each day with calls and an estimate of the cost in dollars, worked out from the tokens and the price of the model. It says it is an estimate and flags calls to a model whose price is not known.
- A Costs section in the admin area shows the last 7, 30 or 90 days.
2026-11-13 11:14:45 +01:00

403 lines
17 KiB
Go

package httpapi
import (
"context"
"fmt"
"net/http"
"slices"
"strings"
"testing"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/naming"
"codeberg.org/squidd/sonda/internal/providers"
"codeberg.org/squidd/sonda/internal/providers/anthropic"
"codeberg.org/squidd/sonda/internal/storage"
)
// askedClaude is a Claude that answers with a fixed name and remembers what it was asked.
type askedClaude struct {
asked []anthropic.Request
keys []string
err error
}
func (c *askedClaude) Name(_ context.Context, key string, r anthropic.Request) (anthropic.Result, error) {
c.asked = append(c.asked, r)
c.keys = append(c.keys, key)
if c.err != nil {
return anthropic.Result{}, c.err
}
return anthropic.Result{Name: "Riff senza sosta", Description: "Hard rock che non hai ancora ascoltato.", Usage: anthropic.Usage{Input: 1000, Output: 500}}, nil
}
const claudeKey = "sk-ant-api03-0123456789abcdefghijklmnopqrstuv"
// newNamingAPI is the mix server with a Claude that is on (with a key) or off.
func newNamingAPI(t *testing.T, claude *askedClaude, on bool) *mixAPI {
t.Helper()
return newMixAPIWith(t, func(d *Deps, db *bun.DB) {
prov := newProviders(db)
key := claudeKey
if _, err := prov.Update(context.Background(), providers.Anthropic, providers.Patch{APIKey: &key}); err != nil {
t.Fatal(err)
}
if on {
yes := true
if _, err := prov.Update(context.Background(), providers.Anthropic, providers.Patch{Enabled: &yes}); err != nil {
t.Fatal(err)
}
}
d.Providers = prov
svc := naming.NewService(prov, claude, storage.NewNamingRepo(db), nil)
svc.SetLedger(storage.NewNamingRepo(db))
d.Naming = svc
})
}
func (m *mixAPI) playlistWithSongs(t *testing.T, title string, n int) int {
t.Helper()
pl := do(t, m.h, "POST", "/api/v1/playlists", m.alice, map[string]any{"name": "tmp"})
id := int(pl.Body["id"].(float64))
rows, err := m.db.Query("SELECT id FROM tracks WHERE title LIKE ? ORDER BY id LIMIT ?", title+"%", n)
if err != nil {
t.Fatal(err)
}
var ids []int64
for rows.Next() {
var tid int64
if err := rows.Scan(&tid); err != nil {
t.Fatal(err)
}
ids = append(ids, tid)
}
rows.Close()
if r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/tracks", id), m.alice, map[string]any{"track_ids": ids}); r.Code != http.StatusOK && r.Code != http.StatusCreated {
t.Fatalf("add: %d %v", r.Code, r.Body)
}
return id
}
func TestNameRoutesNeedASignedInUser(t *testing.T) {
m := newNamingAPI(t, &askedClaude{}, true)
for _, path := range []string{"/api/v1/playlists/1/suggest-name", "/api/v1/mixes/genre:rock/suggest-name", "/api/v1/smart-playlists/1/suggest-name"} {
if r := do(t, m.h, "POST", path, "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("POST %s without a token: %d", path, r.Code)
}
}
}
func TestWithoutClaudeAPlaylistGetsALocalNameAndNothingIsAsked(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, false)
id := m.playlistWithSongs(t, "hard song", 6)
r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, map[string]any{"lang": "en"})
if r.Code != http.StatusOK || r.Body["source"] != "local" || r.Body["name"] != "Hard Rock" {
t.Fatalf("%d %v", r.Code, r.Body)
}
if r.Body["description"] == "" {
t.Errorf("no description: %v", r.Body)
}
if len(c.asked) != 0 {
t.Errorf("Claude was asked with the provider off: %v", c.asked)
}
// With no body at all it is English too, and in Italian a mix is "Mix ...".
if r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, nil); r.Code != http.StatusOK {
t.Errorf("no body: %d", r.Code)
}
mix := do(t, m.h, "POST", "/api/v1/mixes/genre%3Arock/suggest-name", m.alice, map[string]any{"lang": "it"})
if mix.Code != http.StatusOK || mix.Body["name"] != "Mix Rock" || mix.Body["source"] != "local" {
t.Errorf("mix: %d %v", mix.Code, mix.Body)
}
}
func TestClaudeNamesAPlaylistFromFewFacts(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
id := m.playlistWithSongs(t, "hard song", 6)
r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, map[string]any{"lang": "it"})
if r.Code != http.StatusOK || r.Body["source"] != "claude" || r.Body["name"] != "Riff senza sosta" || r.Body["description"] != "Hard rock che non hai ancora ascoltato." {
t.Fatalf("%d %v", r.Code, r.Body)
}
if len(c.asked) != 1 || c.keys[0] != claudeKey {
t.Fatalf("asked %v with %v", c.asked, c.keys)
}
a := c.asked[0]
if a.Kind != "playlist" || a.Language != "it" || a.Tracks != 6 || !slices.Equal(a.Genres, []string{"hard rock"}) || len(a.Artists) == 0 || len(a.Artists) > anthropic.MaxArtists {
t.Errorf("asked %+v", a)
}
// The same facts are not asked twice.
if r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, map[string]any{"lang": "it"}); r.Body["source"] != "claude" {
t.Errorf("the second time: %v", r.Body)
}
if len(c.asked) != 1 {
t.Errorf("%d calls for the same facts", len(c.asked))
}
}
func TestWhenClaudeFailsAPlaylistStillGetsAName(t *testing.T) {
c := &askedClaude{err: providers.ErrUnreachable}
m := newNamingAPI(t, c, true)
id := m.playlistWithSongs(t, "hard song", 3)
r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, nil)
if r.Code != http.StatusOK || r.Body["source"] != "local" || r.Body["name"] != "Hard Rock" {
t.Errorf("%d %v", r.Code, r.Body)
}
}
func TestAMixAndASmartPlaylistAreNamedFromTheirSongsAndRules(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
mix := do(t, m.h, "POST", "/api/v1/mixes/genre%3Ahard%20rock/suggest-name", m.alice, map[string]any{"variant": 2})
if mix.Code != http.StatusOK || mix.Body["source"] != "claude" {
t.Fatalf("mix: %d %v", mix.Code, mix.Body)
}
if got := c.asked[0]; got.Kind != "mix" || got.Tracks != 36 || !slices.Equal(got.Genres, []string{"hard rock"}) {
t.Errorf("mix facts = %+v", got)
}
id := smartMade(t, m, map[string]any{"name": "Fresh", "rules": map[string]any{"genres": []string{"rock"}, "never_played": true}, "limit": 20})
sm := do(t, m.h, "POST", fmt.Sprintf("/api/v1/smart-playlists/%d/suggest-name", id), m.alice, map[string]any{"lang": "en"})
if sm.Code != http.StatusOK || sm.Body["source"] != "claude" {
t.Fatalf("smart: %d %v", sm.Code, sm.Body)
}
if got := c.asked[1]; got.Kind != "smart" || got.Hint != "genres: rock; never played" || got.Tracks != 20 {
t.Errorf("smart facts = %+v", got)
}
}
func TestNameErrorsAreTheOnesOfWhatIsNamed(t *testing.T) {
m := newNamingAPI(t, &askedClaude{}, true)
_, bob := addUser(t, m.h, m.alice, "bob", false)
id := m.playlistWithSongs(t, "hard song", 2)
if r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), bob, nil); r.Code != http.StatusNotFound || errCode(r) != "playlist_not_found" {
t.Errorf("another user's playlist: %d %v", r.Code, r.Body)
}
if r := do(t, m.h, "POST", "/api/v1/playlists/9999/suggest-name", m.alice, nil); r.Code != http.StatusNotFound || errCode(r) != "playlist_not_found" {
t.Errorf("no playlist: %d", r.Code)
}
if r := do(t, m.h, "POST", "/api/v1/mixes/genre%3Apolka/suggest-name", m.alice, nil); r.Code != http.StatusNotFound || errCode(r) != "mix_not_found" {
t.Errorf("no mix: %d", r.Code)
}
if r := do(t, m.h, "POST", "/api/v1/smart-playlists/9999/suggest-name", m.alice, nil); r.Code != http.StatusNotFound || errCode(r) != "smart_playlist_not_found" {
t.Errorf("no smart playlist: %d", r.Code)
}
if r := do(t, m.h, "POST", "/api/v1/playlists/0/suggest-name", m.alice, nil); r.Code != http.StatusBadRequest {
t.Errorf("bad id: %d", r.Code)
}
if r := do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, map[string]any{"variant": -1}); r.Code != http.StatusBadRequest {
t.Errorf("bad variant: %d", r.Code)
}
}
// ---- names made by themselves ----
func playlistNamed(t *testing.T, m *mixAPI, id any) map[string]any {
t.Helper()
r := do(t, m.h, "GET", fmt.Sprintf("/api/v1/playlists/%v", id), m.alice, nil)
if r.Code != http.StatusOK {
t.Fatalf("playlist %v: %d %v", id, r.Code, r.Body)
}
return r.Body
}
func TestWhenClaudeIsOnAKeptMixIsNamedByClaude(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
r := do(t, m.h, "POST", "/api/v1/mixes/genre%3Ahard%20rock/playlist", m.alice, map[string]any{"name": "Mix Hard Rock", "auto_name": true, "lang": "it", "variant": 1})
if r.Code != http.StatusCreated || r.Body["name"] != "Riff senza sosta" {
t.Fatalf("%d %v", r.Code, r.Body)
}
if got := playlistNamed(t, m, r.Body["id"]); got["description"] != "Hard rock che non hai ancora ascoltato." {
t.Errorf("playlist = %v: the line of description is kept too", got)
}
if len(c.asked) != 1 || c.asked[0].Kind != "mix" || c.asked[0].Language != "it" || c.asked[0].Tracks != 36 {
t.Errorf("asked %+v", c.asked)
}
}
func TestTheNameGivenIsKeptWhenNothingAsksForAnAutomaticOne(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
for name, body := range map[string]map[string]any{
"auto_name false": {"name": "My own", "variant": 1},
"auto_name absent": {"name": "My own"},
} {
r := do(t, m.h, "POST", "/api/v1/mixes/genre%3Ahard%20rock/playlist", m.alice, body)
if r.Code != http.StatusCreated || r.Body["name"] != "My own" {
t.Errorf("%s: %d %v", name, r.Code, r.Body)
}
}
if len(c.asked) != 0 {
t.Errorf("Claude was asked for a name nobody wanted: %v", c.asked)
}
}
func TestWithoutClaudeAnAutomaticNameIsTheOneGiven(t *testing.T) {
for name, tc := range map[string]struct {
claude *askedClaude
on bool
}{
"the provider is off": {&askedClaude{}, false},
"Claude fails": {&askedClaude{err: providers.ErrUnreachable}, true},
"Claude declines": {&askedClaude{err: anthropic.ErrNoName}, true},
} {
t.Run(name, func(t *testing.T) {
m := newNamingAPI(t, tc.claude, tc.on)
r := do(t, m.h, "POST", "/api/v1/mixes/genre%3Ahard%20rock/playlist", m.alice, map[string]any{"name": "Hard Rock Mix", "auto_name": true})
if r.Code != http.StatusCreated || r.Body["name"] != "Hard Rock Mix" {
t.Fatalf("%d %v", r.Code, r.Body)
}
if got := playlistNamed(t, m, r.Body["id"]); got["description"] != "" {
t.Errorf("a description came from nowhere: %v", got)
}
if !tc.on && len(tc.claude.asked) != 0 {
t.Error("Claude was asked with the provider off")
}
})
}
}
func TestASmartAndASuggestedPlaylistAreNamedByClaudeWhenKept(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
id := smartMade(t, m, map[string]any{"name": "Fresh", "limit": 8, "rules": map[string]any{"genres": []string{"rock"}, "never_played": true}})
r := do(t, m.h, "POST", fmt.Sprintf("%s/%d/playlist", sp, id), m.alice, map[string]any{"name": "Fresh", "auto_name": true, "lang": "en"})
if r.Code != http.StatusCreated || r.Body["name"] != "Riff senza sosta" {
t.Fatalf("smart: %d %v", r.Code, r.Body)
}
if got := c.asked[0]; got.Kind != "smart" || got.Hint != "genres: rock; never played" {
t.Errorf("smart facts = %+v", got)
}
s := do(t, m.h, "POST", sp+"/suggested/never-played/playlist", m.alice, map[string]any{"name": "Never played", "auto_name": true, "lang": "en"})
if s.Code != http.StatusCreated || s.Body["name"] != "Riff senza sosta" {
t.Fatalf("suggested: %d %v", s.Code, s.Body)
}
if got := c.asked[1]; got.Kind != "smart" || got.Hint != "never played" {
t.Errorf("suggested facts = %+v", got)
}
// Without the request for it, the name stays.
n := do(t, m.h, "POST", sp+"/suggested/never-played/playlist", m.alice, map[string]any{"name": "Never played"})
if n.Body["name"] != "Never played" || len(c.asked) != 2 {
t.Errorf("%v with %d questions", n.Body, len(c.asked))
}
}
func TestASmartPlaylistWithNoNameIsNamedByClaudeWhenItIsOn(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
r := do(t, m.h, "POST", sp, m.alice, map[string]any{"name": " ", "rules": map[string]any{"genres": []string{"hard rock"}}, "limit": 12, "lang": "it"})
if r.Code != http.StatusCreated || r.Body["name"] != "Riff senza sosta" || r.Body["description"] != "Hard rock che non hai ancora ascoltato." {
t.Fatalf("%d %v", r.Code, r.Body)
}
if got := c.asked[0]; got.Kind != "smart" || got.Tracks != 12 || got.Language != "it" || got.Hint != "genres: hard rock" {
t.Errorf("facts = %+v", got)
}
// A name that was given is the name, even with Claude on.
own := do(t, m.h, "POST", sp, m.alice, map[string]any{"name": "Mine", "rules": map[string]any{}})
if own.Body["name"] != "Mine" || len(c.asked) != 1 {
t.Errorf("%v after %d questions", own.Body, len(c.asked))
}
}
func TestASmartPlaylistWithNoNameNeedsOneWhenClaudeCannotMakeIt(t *testing.T) {
for name, m := range map[string]*mixAPI{
"off": newNamingAPI(t, &askedClaude{}, false),
"failing": newNamingAPI(t, &askedClaude{err: providers.ErrUnreachable}, true),
} {
r := do(t, m.h, "POST", sp, m.alice, map[string]any{"name": "", "rules": map[string]any{"genres": []string{"hard rock"}}})
if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("%s: %d %v", name, r.Code, r.Body)
}
if list := do(t, m.h, "GET", sp, m.alice, nil); list.Body["total"] != float64(0) {
t.Errorf("%s: a smart playlist was kept with no name: %v", name, list.Body)
}
}
}
func TestTheClientCanAskWhetherNamesAreAutomatic(t *testing.T) {
on := newNamingAPI(t, &askedClaude{}, true)
if r := do(t, on.h, "GET", "/api/v1/naming", on.alice, nil); r.Code != http.StatusOK || r.Body["auto"] != true {
t.Errorf("on: %d %v", r.Code, r.Body)
}
off := newNamingAPI(t, &askedClaude{}, false)
if r := do(t, off.h, "GET", "/api/v1/naming", off.alice, nil); r.Code != http.StatusOK || r.Body["auto"] != false || len(r.Body) != 1 {
t.Errorf("off: %d %v", r.Code, r.Body)
}
if r := do(t, off.h, "GET", "/api/v1/naming", "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("no token: %d", r.Code)
}
}
// ---- what it costs ----
func TestTheCostsAreForAdminsOnly(t *testing.T) {
m := newNamingAPI(t, &askedClaude{}, true)
_, bob := addUser(t, m.h, m.alice, "bob", false)
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("no token: %d", r.Code)
}
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", bob, nil); r.Code != http.StatusForbidden {
t.Errorf("a user who is not an admin: %d", r.Code)
}
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", m.alice, nil); r.Code != http.StatusOK {
t.Errorf("an admin: %d %v", r.Code, r.Body)
}
}
func TestTheCostsCountWhatWasAskedWhatWasKeptAndWhatFailed(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
empty := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", m.alice, nil)
if empty.Body["days"] != float64(30) || empty.Body["estimate"] != true || empty.Body["model"] != anthropic.Model || len(empty.Body["daily"].([]any)) != 0 {
t.Fatalf("empty report = %v", empty.Body)
}
id := m.playlistWithSongs(t, "hard song", 6)
suggest := func() {
do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, map[string]any{"lang": "it"})
}
suggest() // asked: made
suggest() // the same facts: kept
c.err = providers.ErrRateLimited
do(t, m.h, "POST", "/api/v1/mixes/genre%3Ahard%20rock/suggest-name", m.alice, nil) // asked: failed
r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage?days=7", m.alice, nil)
if r.Code != http.StatusOK || r.Body["days"] != float64(7) {
t.Fatalf("%d %v", r.Code, r.Body)
}
totals := r.Body["totals"].(map[string]any)
if totals["names_made"] != float64(1) || totals["names_kept"] != float64(1) || totals["failures"] != float64(1) ||
totals["input_tokens"] != float64(1000) || totals["output_tokens"] != float64(500) {
t.Errorf("totals = %v", totals)
}
// 1000 tokens in and 500 out at $4 and $20 per million.
if cost := totals["cost_usd"].(float64); cost < 0.01399 || cost > 0.01401 {
t.Errorf("cost = %v", cost)
}
daily := r.Body["daily"].([]any)
if len(daily) != 1 || daily[0].(map[string]any)["names_made"] != float64(1) || daily[0].(map[string]any)["date"] == "" {
t.Errorf("daily = %v", daily)
}
if r.Body["unpriced"] != false {
t.Errorf("unpriced = %v", r.Body["unpriced"])
}
// Nothing of what was sent is in the answer.
raw := fmt.Sprint(r.Body)
for _, never := range []string{"Kiss", "hard rock", "AC/DC"} {
if strings.Contains(raw, never) {
t.Errorf("the report has %q: %s", never, raw)
}
}
}
func TestThePeriodOfTheCostsIsChecked(t *testing.T) {
m := newNamingAPI(t, &askedClaude{}, true)
for _, q := range []string{"?days=-1", "?days=366", "?days=x"} {
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage"+q, m.alice, nil); r.Code != http.StatusBadRequest {
t.Errorf("%s: %d %v", q, r.Code, r.Body)
}
}
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage?days=365", m.alice, nil); r.Code != http.StatusOK {
t.Errorf("a year: %d", r.Code)
}
}