feat(naming): an admin section that shows what Claude costs

- Every question asked of Claude is recorded: when, the kind of list, the model, the tokens read and written (also for an answer that was no name but was billed) and how it ended: a name made, a name taken from the kept ones, or a failure and why. Nothing of the facts that were sent is recorded. Records older than a year are removed (migration 0022).
- GET /providers/anthropic/usage (admin) gives the totals of the last days (1 to 365, 30 by default), a row for each day with calls and an estimate of the cost in dollars, worked out from the tokens and the price of the model. It says it is an estimate and flags calls to a model whose price is not known.
- A Costs section in the admin area shows the last 7, 30 or 90 days.
This commit is contained in:
tgiachi 2026-11-13 11:14:45 +01:00
commit 1b79a7370d
14 changed files with 1181 additions and 13 deletions

View file

@ -112,7 +112,9 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
deps.Playlists = playlists.NewService(storage.NewPlaylistRepo(db))
deps.Mixes = mixes.NewService(storage.NewMixRepo(db), deps.Taste.(mixes.Taster), 20*time.Second)
deps.Smart = smart.NewService(storage.NewSmartRepo(db), deps.Taste.(smart.Taster), 20*time.Second)
deps.Naming = naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil)
namingSvc := naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil)
namingSvc.SetLedger(storage.NewNamingRepo(db))
deps.Naming = namingSvc
imageRepo := storage.NewImageRepo(db)
imageSvc := images.NewService(filepath.Join(t.TempDir(), "data"), imageRepo, imageRepo)
deps.Images, deps.ArtistImages = imageSvc, imageSvc

View file

@ -1085,6 +1085,12 @@ func (c *contract) smartPlaylists(t *testing.T, token string) {
// names asks for names with the Claude provider off, so that the local ones answer.
func (c *contract) names(t *testing.T, token string) {
t.Helper()
c.expect(c.call("GET", "/providers/anthropic/usage", "", nil), 401, "the costs without a token", "unauthorized")
use := c.expect(c.call("GET", "/providers/anthropic/usage?days=7", token, nil), 200, "the costs")
if use["days"] != float64(7) || use["estimate"] != true || len(use["daily"].([]any)) != 0 {
t.Fatalf("costs = %v", use)
}
c.expect(c.call("GET", "/providers/anthropic/usage?days=366", token, nil), 400, "the costs of too many days", "invalid_input")
c.expect(c.call("GET", "/naming", "", nil), 401, "naming status without a token", "unauthorized")
if st := c.expect(c.call("GET", "/naming", token, nil), 200, "naming status"); st["auto"] != false {
t.Fatalf("naming status = %v, want off", st)

View file

@ -1,6 +1,7 @@
package httpapi
import (
"errors"
"net/http"
"github.com/gin-gonic/gin"
@ -140,3 +141,64 @@ func automaticName(c *gin.Context, svc namingService, auto bool, lang, kind stri
}
return s.Name, s.Description
}
type usageTotalsJSON struct {
NamesMade int `json:"names_made"`
NamesKept int `json:"names_kept"`
Failures int `json:"failures"`
InputTokens int64 `json:"input_tokens"`
OutputTokens int64 `json:"output_tokens"`
CacheReadTokens int64 `json:"cache_read_tokens"`
CacheWriteTokens int64 `json:"cache_write_tokens"`
CostUSD float64 `json:"cost_usd"`
}
type usageDayJSON struct {
Date string `json:"date"`
usageTotalsJSON
}
type usageJSON struct {
Days int `json:"days"`
Model string `json:"model"`
Estimate bool `json:"estimate"`
Unpriced bool `json:"unpriced"`
Totals usageTotalsJSON `json:"totals"`
Daily []usageDayJSON `json:"daily"`
}
func toUsageTotals(t naming.Totals) usageTotalsJSON {
return usageTotalsJSON{
NamesMade: t.Made, NamesKept: t.Kept, Failures: t.Failed,
InputTokens: t.Usage.Input, OutputTokens: t.Usage.Output, CacheReadTokens: t.Usage.CacheRead, CacheWriteTokens: t.Usage.CacheWrite,
CostUSD: t.CostUSD,
}
}
// usage says what the calls to Claude cost over the last days (30 by default): the totals, a line for each day that had
// calls, and an estimate in dollars worked out from the tokens. It is for admins, and it holds nothing of the facts that
// were sent.
func (h *namingHandlers) usage(c *gin.Context) {
days, ok := queryInt(c, "days")
if !ok {
return
}
if days == 0 {
days = 30
}
r, err := h.svc.Report(c.Request.Context(), clampInt(days))
if errors.Is(err, naming.ErrInvalidDays) {
writeError(c, http.StatusBadRequest, "invalid_input", "days must be a whole number from 1 to 365")
return
}
if err != nil {
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
return
}
out := usageJSON{Days: r.Days, Model: r.Model, Estimate: r.Estimate, Unpriced: r.Unpriced, Totals: toUsageTotals(r.Totals), Daily: make([]usageDayJSON, len(r.Daily))}
for i, d := range r.Daily {
out.Daily[i] = usageDayJSON{Date: d.Date, usageTotalsJSON: toUsageTotals(d.Totals)}
}
c.JSON(http.StatusOK, out)
}

View file

@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"slices"
"strings"
"testing"
"github.com/uptrace/bun"
@ -28,7 +29,7 @@ func (c *askedClaude) Name(_ context.Context, key string, r anthropic.Request) (
if c.err != nil {
return anthropic.Result{}, c.err
}
return anthropic.Result{Name: "Riff senza sosta", Description: "Hard rock che non hai ancora ascoltato."}, nil
return anthropic.Result{Name: "Riff senza sosta", Description: "Hard rock che non hai ancora ascoltato.", Usage: anthropic.Usage{Input: 1000, Output: 500}}, nil
}
const claudeKey = "sk-ant-api03-0123456789abcdefghijklmnopqrstuv"
@ -49,7 +50,9 @@ func newNamingAPI(t *testing.T, claude *askedClaude, on bool) *mixAPI {
}
}
d.Providers = prov
d.Naming = naming.NewService(prov, claude, storage.NewNamingRepo(db), nil)
svc := naming.NewService(prov, claude, storage.NewNamingRepo(db), nil)
svc.SetLedger(storage.NewNamingRepo(db))
d.Naming = svc
})
}
@ -325,3 +328,76 @@ func TestTheClientCanAskWhetherNamesAreAutomatic(t *testing.T) {
t.Errorf("no token: %d", r.Code)
}
}
// ---- what it costs ----
func TestTheCostsAreForAdminsOnly(t *testing.T) {
m := newNamingAPI(t, &askedClaude{}, true)
_, bob := addUser(t, m.h, m.alice, "bob", false)
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("no token: %d", r.Code)
}
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", bob, nil); r.Code != http.StatusForbidden {
t.Errorf("a user who is not an admin: %d", r.Code)
}
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", m.alice, nil); r.Code != http.StatusOK {
t.Errorf("an admin: %d %v", r.Code, r.Body)
}
}
func TestTheCostsCountWhatWasAskedWhatWasKeptAndWhatFailed(t *testing.T) {
c := &askedClaude{}
m := newNamingAPI(t, c, true)
empty := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", m.alice, nil)
if empty.Body["days"] != float64(30) || empty.Body["estimate"] != true || empty.Body["model"] != anthropic.Model || len(empty.Body["daily"].([]any)) != 0 {
t.Fatalf("empty report = %v", empty.Body)
}
id := m.playlistWithSongs(t, "hard song", 6)
suggest := func() {
do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, map[string]any{"lang": "it"})
}
suggest() // asked: made
suggest() // the same facts: kept
c.err = providers.ErrRateLimited
do(t, m.h, "POST", "/api/v1/mixes/genre%3Ahard%20rock/suggest-name", m.alice, nil) // asked: failed
r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage?days=7", m.alice, nil)
if r.Code != http.StatusOK || r.Body["days"] != float64(7) {
t.Fatalf("%d %v", r.Code, r.Body)
}
totals := r.Body["totals"].(map[string]any)
if totals["names_made"] != float64(1) || totals["names_kept"] != float64(1) || totals["failures"] != float64(1) ||
totals["input_tokens"] != float64(1000) || totals["output_tokens"] != float64(500) {
t.Errorf("totals = %v", totals)
}
// 1000 tokens in and 500 out at $4 and $20 per million.
if cost := totals["cost_usd"].(float64); cost < 0.01399 || cost > 0.01401 {
t.Errorf("cost = %v", cost)
}
daily := r.Body["daily"].([]any)
if len(daily) != 1 || daily[0].(map[string]any)["names_made"] != float64(1) || daily[0].(map[string]any)["date"] == "" {
t.Errorf("daily = %v", daily)
}
if r.Body["unpriced"] != false {
t.Errorf("unpriced = %v", r.Body["unpriced"])
}
// Nothing of what was sent is in the answer.
raw := fmt.Sprint(r.Body)
for _, never := range []string{"Kiss", "hard rock", "AC/DC"} {
if strings.Contains(raw, never) {
t.Errorf("the report has %q: %s", never, raw)
}
}
}
func TestThePeriodOfTheCostsIsChecked(t *testing.T) {
m := newNamingAPI(t, &askedClaude{}, true)
for _, q := range []string{"?days=-1", "?days=366", "?days=x"} {
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage"+q, m.alice, nil); r.Code != http.StatusBadRequest {
t.Errorf("%s: %d %v", q, r.Code, r.Body)
}
}
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage?days=365", m.alice, nil); r.Code != http.StatusOK {
t.Errorf("a year: %d", r.Code)
}
}

View file

@ -113,6 +113,7 @@ type namingService interface {
FactsOf(ctx context.Context, kind string, tracks []catalog.Track, hint string) (naming.Facts, error)
Suggest(ctx context.Context, f naming.Facts, lang string) (naming.Suggestion, error)
Auto(ctx context.Context) bool
Report(ctx context.Context, days int) (naming.Report, error)
}
// smartService is what the handlers need from smart.Service.
@ -354,6 +355,7 @@ func NewRouter(d Deps) *gin.Engine {
// Admins only.
admin := protected.Group("", requireAdmin(d.Auth))
admin.GET("/providers/anthropic/usage", nm.usage)
admin.POST("/library/scan", lib.startScan)
admin.POST("/library/covers", cj.start)
admin.DELETE("/library/covers", cj.stop)

View file

@ -2653,6 +2653,34 @@ paths:
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/providers/anthropic/usage:
get:
tags: [Providers]
operationId: getClaudeUsage
summary: What the Claude provider costs
description: |
Admin only. Every question that Sonda asked of Claude is counted: what it made, what was taken from the names
that were kept (which cost nothing), what failed, and the tokens that were billed, also for a failure that was
billed. From them it works out an **estimate** of the cost in dollars, with the price of the model: the invoice
of Anthropic is the truth. `unpriced` says that some calls were to a model whose price is not known and are not
in the cost. A day is a day in UTC and only the days that had calls are listed, the newest first. Nothing of what
was sent (no artist, no genre, no name) is kept, and records older than a year are removed.
security: [{bearerAuth: []}]
parameters:
- name: days
in: query
description: How many days back, from 1 to 365. 30 when absent or 0.
schema: {type: integer, minimum: 0, maximum: 365, default: 30}
responses:
'200':
description: The totals and the days.
content:
application/json:
schema: {$ref: '#/components/schemas/ClaudeUsage'}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/providers/lidarr/options:
get:
tags: [Providers]
@ -3637,6 +3665,49 @@ components:
name: {type: string}
tracks: {type: integer, description: Playable songs that have the genre.}
ClaudeUsageTotals:
type: object
additionalProperties: false
required: [names_made, names_kept, failures, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, cost_usd]
properties:
names_made: {type: integer, description: Names that Claude made.}
names_kept: {type: integer, description: "Names taken from the ones kept, which cost nothing."}
failures: {type: integer, description: Questions that gave no name.}
input_tokens: {type: integer, format: int64}
output_tokens: {type: integer, format: int64}
cache_read_tokens: {type: integer, format: int64}
cache_write_tokens: {type: integer, format: int64}
cost_usd: {type: number, description: "An estimate in dollars, worked out from the tokens and the price of the model."}
ClaudeUsageDay:
type: object
additionalProperties: false
required: [date, names_made, names_kept, failures, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, cost_usd]
properties:
date: {type: string, format: date, description: "The day, in UTC."}
names_made: {type: integer}
names_kept: {type: integer}
failures: {type: integer}
input_tokens: {type: integer, format: int64}
output_tokens: {type: integer, format: int64}
cache_read_tokens: {type: integer, format: int64}
cache_write_tokens: {type: integer, format: int64}
cost_usd: {type: number, description: "An estimate in dollars."}
ClaudeUsage:
type: object
additionalProperties: false
required: [days, model, estimate, unpriced, totals, daily]
properties:
days: {type: integer}
model: {type: string, description: The model that is asked now.}
estimate: {type: boolean, description: "Always true: the cost is worked out from the tokens, it is not the invoice."}
unpriced: {type: boolean, description: "Some calls were to a model whose price is not known and are not in the cost."}
totals: {$ref: '#/components/schemas/ClaudeUsageTotals'}
daily:
type: array
items: {$ref: '#/components/schemas/ClaudeUsageDay'}
SuggestName:
type: object
additionalProperties: false

View file

@ -11,6 +11,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"slices"
"sort"
@ -66,15 +67,75 @@ type Store interface {
GenreCounts(ctx context.Context, trackIDs []int64) (map[string]int, error)
}
// How a call to Claude ended, and why it failed.
const (
OutcomeMade = "made" // Claude made a name
OutcomeKept = "kept" // the name was one that Claude had made before, taken from the kept ones: nothing was billed
OutcomeFailed = "failed" // Claude gave no name
ReasonNoName = "no_name" // it declined, was cut short or did not answer with a name
ReasonKey = "key"
ReasonRate = "rate"
ReasonUnreachable = "unreachable"
ReasonOther = "other"
)
// Call is one question to Claude, for the account of what it costs. It holds nothing of the facts that were sent: no
// artist, no genre, no name.
type Call struct {
At time.Time
Kind string // "playlist", "mix" or "smart"
Model string
Outcome string
Reason string // for a failure
Usage anthropic.Usage
}
// Meter keeps the account of the calls.
type Meter interface {
Record(ctx context.Context, c Call) error
}
// Service makes the suggestions.
type Service struct {
settings Settings
claude Claude
store Store
meter Meter
ledger Ledger
log *zap.Logger
now func() time.Time
}
// SetMeter makes the service record every question it asks Claude. Call it before the service is used.
func (s *Service) SetMeter(m Meter) { s.meter = m }
// record keeps a call; the account never stops a name.
func (s *Service) record(ctx context.Context, c Call) {
if s.meter == nil {
return
}
c.At, c.Model = s.now(), anthropic.Model
if err := s.meter.Record(ctx, c); err != nil {
s.log.Warn("a call to Claude could not be recorded", zap.Error(err))
}
}
// reasonOf says why a call failed, in one word.
func reasonOf(err error) string {
switch {
case errors.Is(err, anthropic.ErrNoName):
return ReasonNoName
case errors.Is(err, providers.ErrKeyRefused):
return ReasonKey
case errors.Is(err, providers.ErrRateLimited):
return ReasonRate
case errors.Is(err, providers.ErrUnreachable):
return ReasonUnreachable
}
return ReasonOther
}
// NewService builds a Service. log may be nil.
func NewService(settings Settings, claude Claude, store Store, log *zap.Logger) *Service {
if log == nil {
@ -139,6 +200,7 @@ func (s *Service) Suggest(ctx context.Context, f Facts, language string) (Sugges
if got, ok, err := s.store.Get(ctx, key); err != nil {
s.log.Warn("a kept name could not be read", zap.Error(err))
} else if ok {
s.record(ctx, Call{Kind: f.Kind, Outcome: OutcomeKept})
return got, nil
}
res, err := s.claude.Name(ctx, set.APIKey, anthropic.Request{
@ -146,8 +208,10 @@ func (s *Service) Suggest(ctx context.Context, f Facts, language string) (Sugges
})
if err != nil {
s.log.Warn("Claude gave no name", zap.String("provider", providers.Anthropic), zap.Error(err))
s.record(ctx, Call{Kind: f.Kind, Outcome: OutcomeFailed, Reason: reasonOf(err), Usage: res.Usage})
return local(f, language), nil
}
s.record(ctx, Call{Kind: f.Kind, Outcome: OutcomeMade, Usage: res.Usage})
out := Suggestion{Name: res.Name, Description: res.Description, Source: SourceClaude}
if err := s.store.Put(ctx, key, out, s.now()); err != nil {
s.log.Warn("a name could not be kept", zap.Error(err))

View file

@ -72,6 +72,16 @@ func (f *fakeStore) GenreCounts(_ context.Context, ids []int64) (map[string]int,
return out, nil
}
type fakeMeter struct {
calls []Call
err error
}
func (f *fakeMeter) Record(_ context.Context, c Call) error {
f.calls = append(f.calls, c)
return f.err
}
func newService(set fakeSettings, c *fakeClaude, st *fakeStore) *Service {
s := NewService(set, c, st, nil)
s.now = func() time.Time { return time.Unix(1_700_000_000, 0) }
@ -271,3 +281,179 @@ func TestFactsOfNoSongs(t *testing.T) {
t.Errorf("%+v, %v", f, err)
}
}
// ---- what is asked of Claude is recorded ----
func metered(set fakeSettings, c *fakeClaude) (*Service, *fakeMeter) {
m := &fakeMeter{}
s := newService(set, c, newStore())
s.SetMeter(m)
return s, m
}
func TestEveryCallToClaudeIsRecordedWithItsTokens(t *testing.T) {
c := &fakeClaude{res: anthropic.Result{Name: "N", Usage: anthropic.Usage{Input: 120, Output: 45}}}
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
if _, err := s.Suggest(context.Background(), rock, "en"); err != nil {
t.Fatal(err)
}
if len(m.calls) != 1 {
t.Fatalf("calls recorded = %+v", m.calls)
}
got := m.calls[0]
if got.Outcome != OutcomeMade || got.Kind != "mix" || got.Model != anthropic.Model || got.Usage != (anthropic.Usage{Input: 120, Output: 45}) || got.Reason != "" {
t.Errorf("recorded %+v", got)
}
if !got.At.Equal(time.Unix(1_700_000_000, 0)) {
t.Errorf("at = %v", got.At)
}
}
func TestANameTakenFromTheKeptOnesIsRecordedAsSavedAndCostsNothing(t *testing.T) {
c := &fakeClaude{res: anthropic.Result{Name: "N", Usage: anthropic.Usage{Input: 100, Output: 40}}}
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
_, _ = s.Suggest(context.Background(), rock, "en")
_, _ = s.Suggest(context.Background(), rock, "en")
if len(m.calls) != 2 || m.calls[1].Outcome != OutcomeKept || m.calls[1].Usage != (anthropic.Usage{}) {
t.Errorf("recorded %+v", m.calls)
}
}
func TestAFailureIsRecordedWithItsReasonAndTheTokensThatWereBilled(t *testing.T) {
for name, tc := range map[string]struct {
err error
reason string
usage anthropic.Usage
}{
"no name, but billed": {anthropic.ErrNoName, ReasonNoName, anthropic.Usage{Input: 90, Output: 300}},
"key refused": {providers.ErrKeyRefused, ReasonKey, anthropic.Usage{}},
"rate limited": {providers.ErrRateLimited, ReasonRate, anthropic.Usage{}},
"unreachable": {providers.ErrUnreachable, ReasonUnreachable, anthropic.Usage{}},
"anything else": {context.DeadlineExceeded, ReasonOther, anthropic.Usage{}},
} {
t.Run(name, func(t *testing.T) {
c := &fakeClaude{res: anthropic.Result{Usage: tc.usage}, err: tc.err}
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
if got, err := s.Suggest(context.Background(), rock, "en"); err != nil || got.Source != SourceLocal {
t.Fatalf("%+v, %v", got, err)
}
if len(m.calls) != 1 || m.calls[0].Outcome != OutcomeFailed || m.calls[0].Reason != tc.reason || m.calls[0].Usage != tc.usage {
t.Errorf("recorded %+v, want a failure for %q with %+v", m.calls, tc.reason, tc.usage)
}
})
}
}
func TestNothingIsRecordedWhenClaudeIsNotAsked(t *testing.T) {
s, m := metered(fakeSettings{on: false}, &fakeClaude{})
_, _ = s.Suggest(context.Background(), rock, "en")
_, _ = s.Suggest(context.Background(), Facts{Kind: "playlist"}, "en") // nothing to say, nothing asked
if len(m.calls) != 0 {
t.Errorf("recorded %+v", m.calls)
}
}
func TestAMeterThatFailsDoesNotStopTheName(t *testing.T) {
c := &fakeClaude{res: anthropic.Result{Name: "N"}}
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
m.err = errors.New("db down")
got, err := s.Suggest(context.Background(), rock, "en")
if err != nil || got.Name != "N" || got.Source != SourceClaude {
t.Errorf("%+v, %v", got, err)
}
}
// ---- the report of what it costs ----
func TestTheReportSumsTheDaysAndPricesTheTokens(t *testing.T) {
d := func(day int) time.Time { return time.Date(2026, 11, day, 0, 0, 0, 0, time.UTC) }
rows := []DayRow{
{Day: d(12), Model: anthropic.Model, Made: 3, Kept: 5, Failed: 1, Usage: anthropic.Usage{Input: 3000, Output: 1500}},
{Day: d(10), Model: anthropic.Model, Made: 1, Usage: anthropic.Usage{Input: 1000, Output: 500}},
}
r := Summarize(rows, 30, anthropic.Model)
if r.Days != 30 || r.Model != anthropic.Model || !r.Estimate {
t.Errorf("report = %+v", r)
}
if r.Totals.Made != 4 || r.Totals.Kept != 5 || r.Totals.Failed != 1 || r.Totals.Usage.Input != 4000 || r.Totals.Usage.Output != 2000 {
t.Errorf("totals = %+v", r.Totals)
}
// 4000 tokens in and 2000 out at $4 and $20 per million: $0.016 + $0.040.
if r.Totals.CostUSD < 0.05599 || r.Totals.CostUSD > 0.05601 || r.Unpriced {
t.Errorf("cost = %v unpriced = %v", r.Totals.CostUSD, r.Unpriced)
}
if len(r.Daily) != 2 || r.Daily[0].Date != "2026-11-12" || r.Daily[1].Date != "2026-11-10" || r.Daily[0].Made != 3 {
t.Errorf("daily = %+v: the newest day first", r.Daily)
}
if got := r.Daily[1].CostUSD; got < 0.01399 || got > 0.01401 {
t.Errorf("a day's cost = %v", got)
}
}
func TestTwoModelsOnTheSameDayAreOneDayAndAModelWithNoPriceIsSaidSo(t *testing.T) {
day := time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC)
r := Summarize([]DayRow{
{Day: day, Model: anthropic.Model, Made: 1, Usage: anthropic.Usage{Input: 1000, Output: 500}},
{Day: day, Model: "claude-future", Made: 2, Usage: anthropic.Usage{Input: 9000, Output: 9000}},
}, 7, anthropic.Model)
if len(r.Daily) != 1 || r.Daily[0].Made != 3 || r.Daily[0].Usage.Input != 10000 {
t.Fatalf("daily = %+v", r.Daily)
}
if !r.Unpriced || r.Totals.CostUSD < 0.01399 || r.Totals.CostUSD > 0.01401 {
t.Errorf("cost = %v unpriced = %v: only the model that has a price is counted, and the report says there is another", r.Totals.CostUSD, r.Unpriced)
}
}
func TestAReportWithNoCallsIsEmptyNotMissing(t *testing.T) {
r := Summarize(nil, 30, anthropic.Model)
if r.Daily == nil || len(r.Daily) != 0 || r.Totals.CostUSD != 0 || r.Unpriced {
t.Errorf("report = %+v", r)
}
}
type fakeLedger struct {
fakeMeter
rows []DayRow
since time.Time
}
func (f *fakeLedger) Since(_ context.Context, since time.Time) ([]DayRow, error) {
f.since = since
return f.rows, nil
}
func TestTheServiceReadsTheLedgerFromTheStartOfThePeriod(t *testing.T) {
l := &fakeLedger{rows: []DayRow{{Day: time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC), Model: anthropic.Model, Made: 2}}}
s := newService(fakeSettings{}, &fakeClaude{}, newStore())
s.SetLedger(l)
r, err := s.Report(context.Background(), 30)
if err != nil || r.Totals.Made != 2 {
t.Fatalf("%+v, %v", r, err)
}
if want := s.now().Add(-30 * 24 * time.Hour); !l.since.Equal(want) {
t.Errorf("since = %v, want %v", l.since, want)
}
// The ledger is also what records.
s.SetLedger(l)
_, _ = newServiceWithLedger(l).Suggest(context.Background(), rock, "en")
if _, err := s.Report(context.Background(), 0); !errors.Is(err, ErrInvalidDays) {
t.Errorf("0 days: %v", err)
}
if _, err := s.Report(context.Background(), MaxReportDays+1); !errors.Is(err, ErrInvalidDays) {
t.Errorf("too many days: %v", err)
}
}
func newServiceWithLedger(l *fakeLedger) *Service {
s := newService(fakeSettings{on: true, key: "sk-ant-key"}, &fakeClaude{res: anthropic.Result{Name: "N"}}, newStore())
s.SetLedger(l)
return s
}
func TestAServiceWithNoLedgerGivesAnEmptyReport(t *testing.T) {
s := newService(fakeSettings{}, &fakeClaude{}, newStore())
r, err := s.Report(context.Background(), 30)
if err != nil || len(r.Daily) != 0 {
t.Errorf("%+v, %v", r, err)
}
}

117
internal/naming/report.go Normal file
View file

@ -0,0 +1,117 @@
package naming
import (
"context"
"errors"
"fmt"
"sort"
"time"
"codeberg.org/squidd/sonda/internal/providers/anthropic"
)
// The longest period a report covers: the records are kept for a year.
const MaxReportDays = 365
// ErrInvalidDays is what a report gives for a period that is not from 1 to MaxReportDays days.
var ErrInvalidDays = errors.New("naming: days must be from 1 to 365")
// DayRow is what happened on one day (UTC) with one model.
type DayRow struct {
Day time.Time
Model string
Made int
Kept int
Failed int
Usage anthropic.Usage
}
// Ledger is the account of the calls: it records them and tells what happened since a moment.
type Ledger interface {
Meter
// Since returns what happened from that moment on, a row for each day and model, the newest day first.
Since(ctx context.Context, since time.Time) ([]DayRow, error)
}
// SetLedger makes the service record every question it asks Claude and lets it report on them. Call it before the
// service is used.
func (s *Service) SetLedger(l Ledger) {
s.meter = l
s.ledger = l
}
// Totals are the calls of a period, or of a day. CostUSD is an estimate worked out from the tokens and the price of the
// model, not the invoice of Anthropic.
type Totals struct {
Made int // names Claude made
Kept int // names taken from the kept ones, which cost nothing
Failed int
Usage anthropic.Usage
CostUSD float64
}
// Day is the totals of one day (UTC).
type Day struct {
Date string // 2006-01-02
Totals
}
// Report is what Claude costs over a period.
type Report struct {
Days int
Model string // the model that is asked now
Estimate bool // always true: the cost is worked out from the tokens
Unpriced bool // some calls were to a model whose price is not known and are not in the cost
Totals Totals
Daily []Day // the days that had calls, the newest first
}
// Summarize makes the report of the rows: the totals, a line for each day, and the cost of each in dollars.
func Summarize(rows []DayRow, days int, model string) Report {
r := Report{Days: days, Model: model, Estimate: true, Daily: []Day{}}
byDay := map[string]*Day{}
for _, row := range rows {
date := row.Day.UTC().Format("2006-01-02")
d := byDay[date]
if d == nil {
d = &Day{Date: date}
byDay[date] = d
}
var cost float64
if p, ok := anthropic.PriceOf(row.Model); ok {
cost = p.Cost(row.Usage)
} else if row.Usage != (anthropic.Usage{}) {
r.Unpriced = true
}
for _, t := range []*Totals{&d.Totals, &r.Totals} {
t.Made += row.Made
t.Kept += row.Kept
t.Failed += row.Failed
t.Usage.Input += row.Usage.Input
t.Usage.Output += row.Usage.Output
t.Usage.CacheRead += row.Usage.CacheRead
t.Usage.CacheWrite += row.Usage.CacheWrite
t.CostUSD += cost
}
}
for _, d := range byDay {
r.Daily = append(r.Daily, *d)
}
sort.Slice(r.Daily, func(i, j int) bool { return r.Daily[i].Date > r.Daily[j].Date })
return r
}
// Report says what Claude cost over the last days (from 1 to MaxReportDays).
func (s *Service) Report(ctx context.Context, days int) (Report, error) {
if days < 1 || days > MaxReportDays {
return Report{}, ErrInvalidDays
}
if s.ledger == nil {
return Summarize(nil, days, anthropic.Model), nil
}
rows, err := s.ledger.Since(ctx, s.now().Add(-time.Duration(days)*24*time.Hour))
if err != nil {
return Report{}, fmt.Errorf("read what Claude cost: %w", err)
}
return Summarize(rows, days, anthropic.Model), nil
}

View file

@ -57,10 +57,41 @@ type Request struct {
Language string
}
// Result is a name and a line of description.
// Usage is what a call used, in tokens, which is what it is billed by. Cache tokens are the ones read from and written
// to the cache of Anthropic; Sonda sends nothing it asks to cache, so they are zero unless that changes.
type Usage struct {
Input, Output, CacheRead, CacheWrite int64
}
// Result is a name and a line of description. Usage is set also when the call gave no name (ErrNoName), because
// that answer was billed all the same.
type Result struct {
Name string
Description string
Usage Usage
}
// Price is what a model costs, in dollars for each million tokens.
type Price struct {
Input, Output, CacheRead, CacheWrite float64
}
// prices are the published prices of the models that Sonda asks. They are an estimate that the admin page says is one:
// the invoice of Anthropic is the truth. A cache write is 1.25 times the price of an input token, as published.
var prices = map[string]Price{
"claude-opus-5-5": {Input: 4, Output: 20, CacheRead: 0.20, CacheWrite: 5},
}
// PriceOf returns the price of a model, ok=false for one that is not known.
func PriceOf(model string) (Price, bool) {
p, ok := prices[model]
return p, ok
}
// Cost is what the usage costs, in dollars.
func (p Price) Cost(u Usage) float64 {
const million = 1_000_000
return (float64(u.Input)*p.Input + float64(u.Output)*p.Output + float64(u.CacheRead)*p.CacheRead + float64(u.CacheWrite)*p.CacheWrite) / million
}
// Client calls the Claude API. It is safe for concurrent use.
@ -207,8 +238,12 @@ func (c *Client) Name(ctx context.Context, key string, req Request) (Result, err
if err != nil {
return Result{}, failure(err)
}
used := Usage{
Input: resp.Usage.InputTokens, Output: resp.Usage.OutputTokens,
CacheRead: resp.Usage.CacheReadInputTokens, CacheWrite: resp.Usage.CacheCreationInputTokens,
}
if resp.StopReason != sdk.StopReasonEndTurn {
return Result{}, fmt.Errorf("%w: stopped with %s", ErrNoName, strconv.Quote(string(resp.StopReason)))
return Result{Usage: used}, fmt.Errorf("%w: stopped with %s", ErrNoName, strconv.Quote(string(resp.StopReason)))
}
var text string
for _, block := range resp.Content {
@ -221,13 +256,13 @@ func (c *Client) Name(ctx context.Context, key string, req Request) (Result, err
Description string `json:"description"`
}
if err := json.Unmarshal([]byte(text), &out); err != nil {
return Result{}, fmt.Errorf("%w: not JSON", ErrNoName)
return Result{Usage: used}, fmt.Errorf("%w: not JSON", ErrNoName)
}
name := truncate(strings.Trim(clean(out.Name), " \"'“”‘’«»"), MaxNameLen)
if name == "" {
return Result{}, fmt.Errorf("%w: empty", ErrNoName)
return Result{Usage: used}, fmt.Errorf("%w: empty", ErrNoName)
}
return Result{Name: name, Description: truncate(clean(out.Description), MaxDescriptionLen)}, nil
return Result{Name: name, Description: truncate(clean(out.Description), MaxDescriptionLen), Usage: used}, nil
}
// clean takes out control characters and the blanks around and inside, so that a name is one line.

View file

@ -51,7 +51,7 @@ func message(stop, text string) string {
b, _ := json.Marshal(map[string]any{
"id": "msg_1", "type": "message", "role": "assistant", "model": Model, "stop_reason": stop,
"content": []any{map[string]any{"type": "text", "text": text}},
"usage": map[string]any{"input_tokens": 10, "output_tokens": 10},
"usage": map[string]any{"input_tokens": 120, "output_tokens": 45, "cache_read_input_tokens": 7, "cache_creation_input_tokens": 3},
})
return string(b)
}
@ -213,3 +213,39 @@ func TestNothingIsTakenFromTheEnvironmentOfTheServer(t *testing.T) {
t.Error("a header of the environment was sent")
}
}
func TestTheTokensThatWereBilledComeBackWithTheNameAndAlsoWithoutOne(t *testing.T) {
want := Usage{Input: 120, Output: 45, CacheRead: 7, CacheWrite: 3}
ok := newServer(t, 200, message("end_turn", `{"name":"x","description":"y"}`))
got, err := ok.client().Name(context.Background(), "sk-ant-x", req)
if err != nil || got.Usage != want {
t.Errorf("a name: usage = %+v, err = %v, want %+v", got.Usage, err, want)
}
// A refusal and an answer that is no name still cost what they cost.
for name, reply := range map[string]string{"refusal": message("refusal", ""), "cut": message("max_tokens", `{"name":"x"`), "not json": message("end_turn", "nope")} {
s := newServer(t, 200, reply)
res, err := s.client().Name(context.Background(), "sk-ant-x", req)
if !errors.Is(err, ErrNoName) || res.Usage != want {
t.Errorf("%s: usage = %+v, err = %v, want %+v with ErrNoName", name, res.Usage, err, want)
}
}
// A call that did not get through has nothing to bill.
bad := newServer(t, 500, `{"type":"error","error":{"type":"x","message":"y"}}`)
if res, _ := bad.client().Name(context.Background(), "sk-ant-x", req); res.Usage != (Usage{}) {
t.Errorf("a failed call: usage = %+v", res.Usage)
}
}
func TestEveryModelThatIsUsedHasAPrice(t *testing.T) {
p, ok := PriceOf(Model)
if !ok || p.Input <= 0 || p.Output <= 0 {
t.Fatalf("price of %s = %+v, %v", Model, p, ok)
}
// 1000 tokens read and 500 written at $4 and $20 per million.
if got := p.Cost(Usage{Input: 1000, Output: 500}); got < 0.01399 || got > 0.01401 {
t.Errorf("cost = %v, want 0.014", got)
}
if _, ok := PriceOf("claude-nobody"); ok {
t.Error("a model that is not known has a price")
}
}

View file

@ -67,3 +67,49 @@ func (r *NamingRepo) GenreCounts(ctx context.Context, trackIDs []int64) (map[str
}
return out, rows.Err()
}
// keepUsage is how long the account of the calls is kept.
const keepUsage = 365 * 24 * time.Hour
// Record keeps the account of one call to Claude, and forgets the ones that are older than a year.
func (r *NamingRepo) Record(ctx context.Context, c naming.Call) error {
_, err := r.db.ExecContext(ctx,
`INSERT INTO claude_usage (at, kind, model, outcome, reason, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
c.At.Unix(), c.Kind, c.Model, c.Outcome, c.Reason, c.Usage.Input, c.Usage.Output, c.Usage.CacheRead, c.Usage.CacheWrite)
if err != nil {
return fmt.Errorf("record a call: %w", err)
}
if _, err := r.db.ExecContext(ctx, "DELETE FROM claude_usage WHERE at < ?", c.At.Add(-keepUsage).Unix()); err != nil {
return fmt.Errorf("forget old calls: %w", err)
}
return nil
}
// Since returns what happened from a moment on, a row for each day (UTC) and model, the newest day first.
func (r *NamingRepo) Since(ctx context.Context, since time.Time) ([]naming.DayRow, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT at / 86400, model,
SUM(CASE WHEN outcome = 'made' THEN 1 ELSE 0 END),
SUM(CASE WHEN outcome = 'kept' THEN 1 ELSE 0 END),
SUM(CASE WHEN outcome = 'failed' THEN 1 ELSE 0 END),
SUM(input_tokens), SUM(output_tokens), SUM(cache_read_tokens), SUM(cache_write_tokens)
FROM claude_usage WHERE at >= ? GROUP BY 1, 2 ORDER BY 1 DESC, 2`, since.Unix())
if err != nil {
return nil, fmt.Errorf("read what Claude cost: %w", err)
}
defer rows.Close()
out := []naming.DayRow{}
for rows.Next() {
var (
d naming.DayRow
day int64
)
if err := rows.Scan(&day, &d.Model, &d.Made, &d.Kept, &d.Failed, &d.Usage.Input, &d.Usage.Output, &d.Usage.CacheRead, &d.Usage.CacheWrite); err != nil {
return nil, err
}
d.Day = time.Unix(day*86400, 0).UTC()
out = append(out, d)
}
return out, rows.Err()
}

View file

@ -8,6 +8,7 @@ import (
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/naming"
"codeberg.org/squidd/sonda/internal/providers/anthropic"
)
func TestNamingRepoKeepsAndReadsANameAndKeepsTheFirstOneWhenAskedTwice(t *testing.T) {
@ -52,3 +53,74 @@ func TestNamingRepoCountsTheGenresOfSongs(t *testing.T) {
}
})
}
func TestUsageRepoSumsEachDayAndModelAndForgetsWhatIsOlderThanAYear(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
r := NewNamingRepo(db)
day := func(n int) time.Time { return time.Date(2026, 11, n, 15, 30, 0, 0, time.UTC) }
rec := func(at time.Time, kind, model, outcome, reason string, in, out int64) {
t.Helper()
err := r.Record(ctx, naming.Call{At: at, Kind: kind, Model: model, Outcome: outcome, Reason: reason, Usage: anthropic.Usage{Input: in, Output: out, CacheRead: 1, CacheWrite: 2}})
if err != nil {
t.Fatal(err)
}
}
rec(day(12), "mix", "claude-a", naming.OutcomeMade, "", 100, 40)
rec(day(12), "smart", "claude-a", naming.OutcomeMade, "", 200, 60)
rec(day(12), "mix", "claude-a", naming.OutcomeKept, "", 0, 0)
rec(day(12), "mix", "claude-a", naming.OutcomeFailed, naming.ReasonNoName, 50, 300)
rec(day(12), "mix", "claude-b", naming.OutcomeMade, "", 10, 10)
rec(day(10), "playlist", "claude-a", naming.OutcomeFailed, naming.ReasonKey, 0, 0)
rec(day(1), "mix", "claude-a", naming.OutcomeMade, "", 7, 7)
rows, err := r.Since(ctx, day(5))
if err != nil {
t.Fatal(err)
}
if len(rows) != 3 {
t.Fatalf("rows = %+v, want 3: the 12th with two models and the 10th; the 1st is before", rows)
}
first := rows[0]
if !first.Day.Equal(time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC)) || first.Model != "claude-a" || first.Made != 2 || first.Kept != 1 || first.Failed != 1 ||
first.Usage != (anthropic.Usage{Input: 350, Output: 400, CacheRead: 4, CacheWrite: 8}) {
t.Errorf("first = %+v", first)
}
if rows[1].Model != "claude-b" || rows[1].Made != 1 || rows[1].Usage.Input != 10 {
t.Errorf("second = %+v", rows[1])
}
if !rows[2].Day.Equal(time.Date(2026, 11, 10, 0, 0, 0, 0, time.UTC)) || rows[2].Failed != 1 {
t.Errorf("third = %+v", rows[2])
}
if none, _ := r.Since(ctx, day(30)); len(none) != 0 {
t.Errorf("a period with no calls = %+v", none)
}
// A call more than a year after the oldest one makes the oldest go.
if err := r.Record(ctx, naming.Call{At: day(1).Add(366 * 24 * time.Hour), Kind: "mix", Model: "claude-a", Outcome: naming.OutcomeMade}); err != nil {
t.Fatal(err)
}
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM claude_usage WHERE at < ?", day(1).Unix()+1).Scan(&n); err != nil || n != 0 {
t.Errorf("%d records older than a year are left, %v", n, err)
}
if err := db.QueryRow("SELECT COUNT(*) FROM claude_usage").Scan(&n); err != nil || n == 0 {
t.Errorf("the recent ones went too: %d, %v", n, err)
}
})
}
func TestUsageRepoKeepsNothingOfTheFacts(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
cols, err := db.Query("SELECT * FROM claude_usage LIMIT 0")
if err != nil {
t.Fatal(err)
}
defer cols.Close()
names, _ := cols.Columns()
want := []string{"id", "at", "kind", "model", "outcome", "reason", "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"}
if len(names) != len(want) {
t.Errorf("columns = %v, want %v: nothing but the account", names, want)
}
})
}

View file

@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) {
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil {
t.Fatal(err)
}
if applied != 21 {
t.Errorf("applied migrations = %d, want 21", applied)
if applied != 22 {
t.Errorf("applied migrations = %d, want 22", applied)
}
})
}