feat(naming): an admin section that shows what Claude costs
- Every question asked of Claude is recorded: when, the kind of list, the model, the tokens read and written (also for an answer that was no name but was billed) and how it ended: a name made, a name taken from the kept ones, or a failure and why. Nothing of the facts that were sent is recorded. Records older than a year are removed (migration 0022). - GET /providers/anthropic/usage (admin) gives the totals of the last days (1 to 365, 30 by default), a row for each day with calls and an estimate of the cost in dollars, worked out from the tokens and the price of the model. It says it is an estimate and flags calls to a model whose price is not known. - A Costs section in the admin area shows the last 7, 30 or 90 days.
This commit is contained in:
parent
84b6937bbf
commit
1b79a7370d
14 changed files with 1181 additions and 13 deletions
|
|
@ -112,7 +112,9 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
|
|||
deps.Playlists = playlists.NewService(storage.NewPlaylistRepo(db))
|
||||
deps.Mixes = mixes.NewService(storage.NewMixRepo(db), deps.Taste.(mixes.Taster), 20*time.Second)
|
||||
deps.Smart = smart.NewService(storage.NewSmartRepo(db), deps.Taste.(smart.Taster), 20*time.Second)
|
||||
deps.Naming = naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil)
|
||||
namingSvc := naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil)
|
||||
namingSvc.SetLedger(storage.NewNamingRepo(db))
|
||||
deps.Naming = namingSvc
|
||||
imageRepo := storage.NewImageRepo(db)
|
||||
imageSvc := images.NewService(filepath.Join(t.TempDir(), "data"), imageRepo, imageRepo)
|
||||
deps.Images, deps.ArtistImages = imageSvc, imageSvc
|
||||
|
|
|
|||
|
|
@ -1085,6 +1085,12 @@ func (c *contract) smartPlaylists(t *testing.T, token string) {
|
|||
// names asks for names with the Claude provider off, so that the local ones answer.
|
||||
func (c *contract) names(t *testing.T, token string) {
|
||||
t.Helper()
|
||||
c.expect(c.call("GET", "/providers/anthropic/usage", "", nil), 401, "the costs without a token", "unauthorized")
|
||||
use := c.expect(c.call("GET", "/providers/anthropic/usage?days=7", token, nil), 200, "the costs")
|
||||
if use["days"] != float64(7) || use["estimate"] != true || len(use["daily"].([]any)) != 0 {
|
||||
t.Fatalf("costs = %v", use)
|
||||
}
|
||||
c.expect(c.call("GET", "/providers/anthropic/usage?days=366", token, nil), 400, "the costs of too many days", "invalid_input")
|
||||
c.expect(c.call("GET", "/naming", "", nil), 401, "naming status without a token", "unauthorized")
|
||||
if st := c.expect(c.call("GET", "/naming", token, nil), 200, "naming status"); st["auto"] != false {
|
||||
t.Fatalf("naming status = %v, want off", st)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
|
@ -140,3 +141,64 @@ func automaticName(c *gin.Context, svc namingService, auto bool, lang, kind stri
|
|||
}
|
||||
return s.Name, s.Description
|
||||
}
|
||||
|
||||
type usageTotalsJSON struct {
|
||||
NamesMade int `json:"names_made"`
|
||||
NamesKept int `json:"names_kept"`
|
||||
Failures int `json:"failures"`
|
||||
InputTokens int64 `json:"input_tokens"`
|
||||
OutputTokens int64 `json:"output_tokens"`
|
||||
CacheReadTokens int64 `json:"cache_read_tokens"`
|
||||
CacheWriteTokens int64 `json:"cache_write_tokens"`
|
||||
CostUSD float64 `json:"cost_usd"`
|
||||
}
|
||||
|
||||
type usageDayJSON struct {
|
||||
Date string `json:"date"`
|
||||
usageTotalsJSON
|
||||
}
|
||||
|
||||
type usageJSON struct {
|
||||
Days int `json:"days"`
|
||||
Model string `json:"model"`
|
||||
Estimate bool `json:"estimate"`
|
||||
Unpriced bool `json:"unpriced"`
|
||||
Totals usageTotalsJSON `json:"totals"`
|
||||
Daily []usageDayJSON `json:"daily"`
|
||||
}
|
||||
|
||||
func toUsageTotals(t naming.Totals) usageTotalsJSON {
|
||||
return usageTotalsJSON{
|
||||
NamesMade: t.Made, NamesKept: t.Kept, Failures: t.Failed,
|
||||
InputTokens: t.Usage.Input, OutputTokens: t.Usage.Output, CacheReadTokens: t.Usage.CacheRead, CacheWriteTokens: t.Usage.CacheWrite,
|
||||
CostUSD: t.CostUSD,
|
||||
}
|
||||
}
|
||||
|
||||
// usage says what the calls to Claude cost over the last days (30 by default): the totals, a line for each day that had
|
||||
// calls, and an estimate in dollars worked out from the tokens. It is for admins, and it holds nothing of the facts that
|
||||
// were sent.
|
||||
func (h *namingHandlers) usage(c *gin.Context) {
|
||||
days, ok := queryInt(c, "days")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if days == 0 {
|
||||
days = 30
|
||||
}
|
||||
r, err := h.svc.Report(c.Request.Context(), clampInt(days))
|
||||
if errors.Is(err, naming.ErrInvalidDays) {
|
||||
writeError(c, http.StatusBadRequest, "invalid_input", "days must be a whole number from 1 to 365")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
_ = c.Error(err)
|
||||
writeError(c, http.StatusInternalServerError, "internal", "internal error")
|
||||
return
|
||||
}
|
||||
out := usageJSON{Days: r.Days, Model: r.Model, Estimate: r.Estimate, Unpriced: r.Unpriced, Totals: toUsageTotals(r.Totals), Daily: make([]usageDayJSON, len(r.Daily))}
|
||||
for i, d := range r.Daily {
|
||||
out.Daily[i] = usageDayJSON{Date: d.Date, usageTotalsJSON: toUsageTotals(d.Totals)}
|
||||
}
|
||||
c.JSON(http.StatusOK, out)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
|
@ -28,7 +29,7 @@ func (c *askedClaude) Name(_ context.Context, key string, r anthropic.Request) (
|
|||
if c.err != nil {
|
||||
return anthropic.Result{}, c.err
|
||||
}
|
||||
return anthropic.Result{Name: "Riff senza sosta", Description: "Hard rock che non hai ancora ascoltato."}, nil
|
||||
return anthropic.Result{Name: "Riff senza sosta", Description: "Hard rock che non hai ancora ascoltato.", Usage: anthropic.Usage{Input: 1000, Output: 500}}, nil
|
||||
}
|
||||
|
||||
const claudeKey = "sk-ant-api03-0123456789abcdefghijklmnopqrstuv"
|
||||
|
|
@ -49,7 +50,9 @@ func newNamingAPI(t *testing.T, claude *askedClaude, on bool) *mixAPI {
|
|||
}
|
||||
}
|
||||
d.Providers = prov
|
||||
d.Naming = naming.NewService(prov, claude, storage.NewNamingRepo(db), nil)
|
||||
svc := naming.NewService(prov, claude, storage.NewNamingRepo(db), nil)
|
||||
svc.SetLedger(storage.NewNamingRepo(db))
|
||||
d.Naming = svc
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -325,3 +328,76 @@ func TestTheClientCanAskWhetherNamesAreAutomatic(t *testing.T) {
|
|||
t.Errorf("no token: %d", r.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- what it costs ----
|
||||
|
||||
func TestTheCostsAreForAdminsOnly(t *testing.T) {
|
||||
m := newNamingAPI(t, &askedClaude{}, true)
|
||||
_, bob := addUser(t, m.h, m.alice, "bob", false)
|
||||
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", "", nil); r.Code != http.StatusUnauthorized {
|
||||
t.Errorf("no token: %d", r.Code)
|
||||
}
|
||||
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", bob, nil); r.Code != http.StatusForbidden {
|
||||
t.Errorf("a user who is not an admin: %d", r.Code)
|
||||
}
|
||||
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", m.alice, nil); r.Code != http.StatusOK {
|
||||
t.Errorf("an admin: %d %v", r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCostsCountWhatWasAskedWhatWasKeptAndWhatFailed(t *testing.T) {
|
||||
c := &askedClaude{}
|
||||
m := newNamingAPI(t, c, true)
|
||||
empty := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage", m.alice, nil)
|
||||
if empty.Body["days"] != float64(30) || empty.Body["estimate"] != true || empty.Body["model"] != anthropic.Model || len(empty.Body["daily"].([]any)) != 0 {
|
||||
t.Fatalf("empty report = %v", empty.Body)
|
||||
}
|
||||
id := m.playlistWithSongs(t, "hard song", 6)
|
||||
suggest := func() {
|
||||
do(t, m.h, "POST", fmt.Sprintf("/api/v1/playlists/%d/suggest-name", id), m.alice, map[string]any{"lang": "it"})
|
||||
}
|
||||
suggest() // asked: made
|
||||
suggest() // the same facts: kept
|
||||
c.err = providers.ErrRateLimited
|
||||
do(t, m.h, "POST", "/api/v1/mixes/genre%3Ahard%20rock/suggest-name", m.alice, nil) // asked: failed
|
||||
|
||||
r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage?days=7", m.alice, nil)
|
||||
if r.Code != http.StatusOK || r.Body["days"] != float64(7) {
|
||||
t.Fatalf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
totals := r.Body["totals"].(map[string]any)
|
||||
if totals["names_made"] != float64(1) || totals["names_kept"] != float64(1) || totals["failures"] != float64(1) ||
|
||||
totals["input_tokens"] != float64(1000) || totals["output_tokens"] != float64(500) {
|
||||
t.Errorf("totals = %v", totals)
|
||||
}
|
||||
// 1000 tokens in and 500 out at $4 and $20 per million.
|
||||
if cost := totals["cost_usd"].(float64); cost < 0.01399 || cost > 0.01401 {
|
||||
t.Errorf("cost = %v", cost)
|
||||
}
|
||||
daily := r.Body["daily"].([]any)
|
||||
if len(daily) != 1 || daily[0].(map[string]any)["names_made"] != float64(1) || daily[0].(map[string]any)["date"] == "" {
|
||||
t.Errorf("daily = %v", daily)
|
||||
}
|
||||
if r.Body["unpriced"] != false {
|
||||
t.Errorf("unpriced = %v", r.Body["unpriced"])
|
||||
}
|
||||
// Nothing of what was sent is in the answer.
|
||||
raw := fmt.Sprint(r.Body)
|
||||
for _, never := range []string{"Kiss", "hard rock", "AC/DC"} {
|
||||
if strings.Contains(raw, never) {
|
||||
t.Errorf("the report has %q: %s", never, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestThePeriodOfTheCostsIsChecked(t *testing.T) {
|
||||
m := newNamingAPI(t, &askedClaude{}, true)
|
||||
for _, q := range []string{"?days=-1", "?days=366", "?days=x"} {
|
||||
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage"+q, m.alice, nil); r.Code != http.StatusBadRequest {
|
||||
t.Errorf("%s: %d %v", q, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if r := do(t, m.h, "GET", "/api/v1/providers/anthropic/usage?days=365", m.alice, nil); r.Code != http.StatusOK {
|
||||
t.Errorf("a year: %d", r.Code)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ type namingService interface {
|
|||
FactsOf(ctx context.Context, kind string, tracks []catalog.Track, hint string) (naming.Facts, error)
|
||||
Suggest(ctx context.Context, f naming.Facts, lang string) (naming.Suggestion, error)
|
||||
Auto(ctx context.Context) bool
|
||||
Report(ctx context.Context, days int) (naming.Report, error)
|
||||
}
|
||||
|
||||
// smartService is what the handlers need from smart.Service.
|
||||
|
|
@ -354,6 +355,7 @@ func NewRouter(d Deps) *gin.Engine {
|
|||
|
||||
// Admins only.
|
||||
admin := protected.Group("", requireAdmin(d.Auth))
|
||||
admin.GET("/providers/anthropic/usage", nm.usage)
|
||||
admin.POST("/library/scan", lib.startScan)
|
||||
admin.POST("/library/covers", cj.start)
|
||||
admin.DELETE("/library/covers", cj.stop)
|
||||
|
|
|
|||
|
|
@ -2653,6 +2653,34 @@ paths:
|
|||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/providers/anthropic/usage:
|
||||
get:
|
||||
tags: [Providers]
|
||||
operationId: getClaudeUsage
|
||||
summary: What the Claude provider costs
|
||||
description: |
|
||||
Admin only. Every question that Sonda asked of Claude is counted: what it made, what was taken from the names
|
||||
that were kept (which cost nothing), what failed, and the tokens that were billed, also for a failure that was
|
||||
billed. From them it works out an **estimate** of the cost in dollars, with the price of the model: the invoice
|
||||
of Anthropic is the truth. `unpriced` says that some calls were to a model whose price is not known and are not
|
||||
in the cost. A day is a day in UTC and only the days that had calls are listed, the newest first. Nothing of what
|
||||
was sent (no artist, no genre, no name) is kept, and records older than a year are removed.
|
||||
security: [{bearerAuth: []}]
|
||||
parameters:
|
||||
- name: days
|
||||
in: query
|
||||
description: How many days back, from 1 to 365. 30 when absent or 0.
|
||||
schema: {type: integer, minimum: 0, maximum: 365, default: 30}
|
||||
responses:
|
||||
'200':
|
||||
description: The totals and the days.
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/ClaudeUsage'}
|
||||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/providers/lidarr/options:
|
||||
get:
|
||||
tags: [Providers]
|
||||
|
|
@ -3637,6 +3665,49 @@ components:
|
|||
name: {type: string}
|
||||
tracks: {type: integer, description: Playable songs that have the genre.}
|
||||
|
||||
ClaudeUsageTotals:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [names_made, names_kept, failures, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, cost_usd]
|
||||
properties:
|
||||
names_made: {type: integer, description: Names that Claude made.}
|
||||
names_kept: {type: integer, description: "Names taken from the ones kept, which cost nothing."}
|
||||
failures: {type: integer, description: Questions that gave no name.}
|
||||
input_tokens: {type: integer, format: int64}
|
||||
output_tokens: {type: integer, format: int64}
|
||||
cache_read_tokens: {type: integer, format: int64}
|
||||
cache_write_tokens: {type: integer, format: int64}
|
||||
cost_usd: {type: number, description: "An estimate in dollars, worked out from the tokens and the price of the model."}
|
||||
|
||||
ClaudeUsageDay:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [date, names_made, names_kept, failures, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, cost_usd]
|
||||
properties:
|
||||
date: {type: string, format: date, description: "The day, in UTC."}
|
||||
names_made: {type: integer}
|
||||
names_kept: {type: integer}
|
||||
failures: {type: integer}
|
||||
input_tokens: {type: integer, format: int64}
|
||||
output_tokens: {type: integer, format: int64}
|
||||
cache_read_tokens: {type: integer, format: int64}
|
||||
cache_write_tokens: {type: integer, format: int64}
|
||||
cost_usd: {type: number, description: "An estimate in dollars."}
|
||||
|
||||
ClaudeUsage:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [days, model, estimate, unpriced, totals, daily]
|
||||
properties:
|
||||
days: {type: integer}
|
||||
model: {type: string, description: The model that is asked now.}
|
||||
estimate: {type: boolean, description: "Always true: the cost is worked out from the tokens, it is not the invoice."}
|
||||
unpriced: {type: boolean, description: "Some calls were to a model whose price is not known and are not in the cost."}
|
||||
totals: {$ref: '#/components/schemas/ClaudeUsageTotals'}
|
||||
daily:
|
||||
type: array
|
||||
items: {$ref: '#/components/schemas/ClaudeUsageDay'}
|
||||
|
||||
SuggestName:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
|
|
@ -66,15 +67,75 @@ type Store interface {
|
|||
GenreCounts(ctx context.Context, trackIDs []int64) (map[string]int, error)
|
||||
}
|
||||
|
||||
// How a call to Claude ended, and why it failed.
|
||||
const (
|
||||
OutcomeMade = "made" // Claude made a name
|
||||
OutcomeKept = "kept" // the name was one that Claude had made before, taken from the kept ones: nothing was billed
|
||||
OutcomeFailed = "failed" // Claude gave no name
|
||||
|
||||
ReasonNoName = "no_name" // it declined, was cut short or did not answer with a name
|
||||
ReasonKey = "key"
|
||||
ReasonRate = "rate"
|
||||
ReasonUnreachable = "unreachable"
|
||||
ReasonOther = "other"
|
||||
)
|
||||
|
||||
// Call is one question to Claude, for the account of what it costs. It holds nothing of the facts that were sent: no
|
||||
// artist, no genre, no name.
|
||||
type Call struct {
|
||||
At time.Time
|
||||
Kind string // "playlist", "mix" or "smart"
|
||||
Model string
|
||||
Outcome string
|
||||
Reason string // for a failure
|
||||
Usage anthropic.Usage
|
||||
}
|
||||
|
||||
// Meter keeps the account of the calls.
|
||||
type Meter interface {
|
||||
Record(ctx context.Context, c Call) error
|
||||
}
|
||||
|
||||
// Service makes the suggestions.
|
||||
type Service struct {
|
||||
settings Settings
|
||||
claude Claude
|
||||
store Store
|
||||
meter Meter
|
||||
ledger Ledger
|
||||
log *zap.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// SetMeter makes the service record every question it asks Claude. Call it before the service is used.
|
||||
func (s *Service) SetMeter(m Meter) { s.meter = m }
|
||||
|
||||
// record keeps a call; the account never stops a name.
|
||||
func (s *Service) record(ctx context.Context, c Call) {
|
||||
if s.meter == nil {
|
||||
return
|
||||
}
|
||||
c.At, c.Model = s.now(), anthropic.Model
|
||||
if err := s.meter.Record(ctx, c); err != nil {
|
||||
s.log.Warn("a call to Claude could not be recorded", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// reasonOf says why a call failed, in one word.
|
||||
func reasonOf(err error) string {
|
||||
switch {
|
||||
case errors.Is(err, anthropic.ErrNoName):
|
||||
return ReasonNoName
|
||||
case errors.Is(err, providers.ErrKeyRefused):
|
||||
return ReasonKey
|
||||
case errors.Is(err, providers.ErrRateLimited):
|
||||
return ReasonRate
|
||||
case errors.Is(err, providers.ErrUnreachable):
|
||||
return ReasonUnreachable
|
||||
}
|
||||
return ReasonOther
|
||||
}
|
||||
|
||||
// NewService builds a Service. log may be nil.
|
||||
func NewService(settings Settings, claude Claude, store Store, log *zap.Logger) *Service {
|
||||
if log == nil {
|
||||
|
|
@ -139,6 +200,7 @@ func (s *Service) Suggest(ctx context.Context, f Facts, language string) (Sugges
|
|||
if got, ok, err := s.store.Get(ctx, key); err != nil {
|
||||
s.log.Warn("a kept name could not be read", zap.Error(err))
|
||||
} else if ok {
|
||||
s.record(ctx, Call{Kind: f.Kind, Outcome: OutcomeKept})
|
||||
return got, nil
|
||||
}
|
||||
res, err := s.claude.Name(ctx, set.APIKey, anthropic.Request{
|
||||
|
|
@ -146,8 +208,10 @@ func (s *Service) Suggest(ctx context.Context, f Facts, language string) (Sugges
|
|||
})
|
||||
if err != nil {
|
||||
s.log.Warn("Claude gave no name", zap.String("provider", providers.Anthropic), zap.Error(err))
|
||||
s.record(ctx, Call{Kind: f.Kind, Outcome: OutcomeFailed, Reason: reasonOf(err), Usage: res.Usage})
|
||||
return local(f, language), nil
|
||||
}
|
||||
s.record(ctx, Call{Kind: f.Kind, Outcome: OutcomeMade, Usage: res.Usage})
|
||||
out := Suggestion{Name: res.Name, Description: res.Description, Source: SourceClaude}
|
||||
if err := s.store.Put(ctx, key, out, s.now()); err != nil {
|
||||
s.log.Warn("a name could not be kept", zap.Error(err))
|
||||
|
|
|
|||
|
|
@ -72,6 +72,16 @@ func (f *fakeStore) GenreCounts(_ context.Context, ids []int64) (map[string]int,
|
|||
return out, nil
|
||||
}
|
||||
|
||||
type fakeMeter struct {
|
||||
calls []Call
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeMeter) Record(_ context.Context, c Call) error {
|
||||
f.calls = append(f.calls, c)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func newService(set fakeSettings, c *fakeClaude, st *fakeStore) *Service {
|
||||
s := NewService(set, c, st, nil)
|
||||
s.now = func() time.Time { return time.Unix(1_700_000_000, 0) }
|
||||
|
|
@ -271,3 +281,179 @@ func TestFactsOfNoSongs(t *testing.T) {
|
|||
t.Errorf("%+v, %v", f, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- what is asked of Claude is recorded ----
|
||||
|
||||
func metered(set fakeSettings, c *fakeClaude) (*Service, *fakeMeter) {
|
||||
m := &fakeMeter{}
|
||||
s := newService(set, c, newStore())
|
||||
s.SetMeter(m)
|
||||
return s, m
|
||||
}
|
||||
|
||||
func TestEveryCallToClaudeIsRecordedWithItsTokens(t *testing.T) {
|
||||
c := &fakeClaude{res: anthropic.Result{Name: "N", Usage: anthropic.Usage{Input: 120, Output: 45}}}
|
||||
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
|
||||
if _, err := s.Suggest(context.Background(), rock, "en"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.calls) != 1 {
|
||||
t.Fatalf("calls recorded = %+v", m.calls)
|
||||
}
|
||||
got := m.calls[0]
|
||||
if got.Outcome != OutcomeMade || got.Kind != "mix" || got.Model != anthropic.Model || got.Usage != (anthropic.Usage{Input: 120, Output: 45}) || got.Reason != "" {
|
||||
t.Errorf("recorded %+v", got)
|
||||
}
|
||||
if !got.At.Equal(time.Unix(1_700_000_000, 0)) {
|
||||
t.Errorf("at = %v", got.At)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANameTakenFromTheKeptOnesIsRecordedAsSavedAndCostsNothing(t *testing.T) {
|
||||
c := &fakeClaude{res: anthropic.Result{Name: "N", Usage: anthropic.Usage{Input: 100, Output: 40}}}
|
||||
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
|
||||
_, _ = s.Suggest(context.Background(), rock, "en")
|
||||
_, _ = s.Suggest(context.Background(), rock, "en")
|
||||
if len(m.calls) != 2 || m.calls[1].Outcome != OutcomeKept || m.calls[1].Usage != (anthropic.Usage{}) {
|
||||
t.Errorf("recorded %+v", m.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailureIsRecordedWithItsReasonAndTheTokensThatWereBilled(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
err error
|
||||
reason string
|
||||
usage anthropic.Usage
|
||||
}{
|
||||
"no name, but billed": {anthropic.ErrNoName, ReasonNoName, anthropic.Usage{Input: 90, Output: 300}},
|
||||
"key refused": {providers.ErrKeyRefused, ReasonKey, anthropic.Usage{}},
|
||||
"rate limited": {providers.ErrRateLimited, ReasonRate, anthropic.Usage{}},
|
||||
"unreachable": {providers.ErrUnreachable, ReasonUnreachable, anthropic.Usage{}},
|
||||
"anything else": {context.DeadlineExceeded, ReasonOther, anthropic.Usage{}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
c := &fakeClaude{res: anthropic.Result{Usage: tc.usage}, err: tc.err}
|
||||
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
|
||||
if got, err := s.Suggest(context.Background(), rock, "en"); err != nil || got.Source != SourceLocal {
|
||||
t.Fatalf("%+v, %v", got, err)
|
||||
}
|
||||
if len(m.calls) != 1 || m.calls[0].Outcome != OutcomeFailed || m.calls[0].Reason != tc.reason || m.calls[0].Usage != tc.usage {
|
||||
t.Errorf("recorded %+v, want a failure for %q with %+v", m.calls, tc.reason, tc.usage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingIsRecordedWhenClaudeIsNotAsked(t *testing.T) {
|
||||
s, m := metered(fakeSettings{on: false}, &fakeClaude{})
|
||||
_, _ = s.Suggest(context.Background(), rock, "en")
|
||||
_, _ = s.Suggest(context.Background(), Facts{Kind: "playlist"}, "en") // nothing to say, nothing asked
|
||||
if len(m.calls) != 0 {
|
||||
t.Errorf("recorded %+v", m.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMeterThatFailsDoesNotStopTheName(t *testing.T) {
|
||||
c := &fakeClaude{res: anthropic.Result{Name: "N"}}
|
||||
s, m := metered(fakeSettings{on: true, key: "sk-ant-key"}, c)
|
||||
m.err = errors.New("db down")
|
||||
got, err := s.Suggest(context.Background(), rock, "en")
|
||||
if err != nil || got.Name != "N" || got.Source != SourceClaude {
|
||||
t.Errorf("%+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the report of what it costs ----
|
||||
|
||||
func TestTheReportSumsTheDaysAndPricesTheTokens(t *testing.T) {
|
||||
d := func(day int) time.Time { return time.Date(2026, 11, day, 0, 0, 0, 0, time.UTC) }
|
||||
rows := []DayRow{
|
||||
{Day: d(12), Model: anthropic.Model, Made: 3, Kept: 5, Failed: 1, Usage: anthropic.Usage{Input: 3000, Output: 1500}},
|
||||
{Day: d(10), Model: anthropic.Model, Made: 1, Usage: anthropic.Usage{Input: 1000, Output: 500}},
|
||||
}
|
||||
r := Summarize(rows, 30, anthropic.Model)
|
||||
if r.Days != 30 || r.Model != anthropic.Model || !r.Estimate {
|
||||
t.Errorf("report = %+v", r)
|
||||
}
|
||||
if r.Totals.Made != 4 || r.Totals.Kept != 5 || r.Totals.Failed != 1 || r.Totals.Usage.Input != 4000 || r.Totals.Usage.Output != 2000 {
|
||||
t.Errorf("totals = %+v", r.Totals)
|
||||
}
|
||||
// 4000 tokens in and 2000 out at $4 and $20 per million: $0.016 + $0.040.
|
||||
if r.Totals.CostUSD < 0.05599 || r.Totals.CostUSD > 0.05601 || r.Unpriced {
|
||||
t.Errorf("cost = %v unpriced = %v", r.Totals.CostUSD, r.Unpriced)
|
||||
}
|
||||
if len(r.Daily) != 2 || r.Daily[0].Date != "2026-11-12" || r.Daily[1].Date != "2026-11-10" || r.Daily[0].Made != 3 {
|
||||
t.Errorf("daily = %+v: the newest day first", r.Daily)
|
||||
}
|
||||
if got := r.Daily[1].CostUSD; got < 0.01399 || got > 0.01401 {
|
||||
t.Errorf("a day's cost = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoModelsOnTheSameDayAreOneDayAndAModelWithNoPriceIsSaidSo(t *testing.T) {
|
||||
day := time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC)
|
||||
r := Summarize([]DayRow{
|
||||
{Day: day, Model: anthropic.Model, Made: 1, Usage: anthropic.Usage{Input: 1000, Output: 500}},
|
||||
{Day: day, Model: "claude-future", Made: 2, Usage: anthropic.Usage{Input: 9000, Output: 9000}},
|
||||
}, 7, anthropic.Model)
|
||||
if len(r.Daily) != 1 || r.Daily[0].Made != 3 || r.Daily[0].Usage.Input != 10000 {
|
||||
t.Fatalf("daily = %+v", r.Daily)
|
||||
}
|
||||
if !r.Unpriced || r.Totals.CostUSD < 0.01399 || r.Totals.CostUSD > 0.01401 {
|
||||
t.Errorf("cost = %v unpriced = %v: only the model that has a price is counted, and the report says there is another", r.Totals.CostUSD, r.Unpriced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReportWithNoCallsIsEmptyNotMissing(t *testing.T) {
|
||||
r := Summarize(nil, 30, anthropic.Model)
|
||||
if r.Daily == nil || len(r.Daily) != 0 || r.Totals.CostUSD != 0 || r.Unpriced {
|
||||
t.Errorf("report = %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeLedger struct {
|
||||
fakeMeter
|
||||
rows []DayRow
|
||||
since time.Time
|
||||
}
|
||||
|
||||
func (f *fakeLedger) Since(_ context.Context, since time.Time) ([]DayRow, error) {
|
||||
f.since = since
|
||||
return f.rows, nil
|
||||
}
|
||||
|
||||
func TestTheServiceReadsTheLedgerFromTheStartOfThePeriod(t *testing.T) {
|
||||
l := &fakeLedger{rows: []DayRow{{Day: time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC), Model: anthropic.Model, Made: 2}}}
|
||||
s := newService(fakeSettings{}, &fakeClaude{}, newStore())
|
||||
s.SetLedger(l)
|
||||
r, err := s.Report(context.Background(), 30)
|
||||
if err != nil || r.Totals.Made != 2 {
|
||||
t.Fatalf("%+v, %v", r, err)
|
||||
}
|
||||
if want := s.now().Add(-30 * 24 * time.Hour); !l.since.Equal(want) {
|
||||
t.Errorf("since = %v, want %v", l.since, want)
|
||||
}
|
||||
// The ledger is also what records.
|
||||
s.SetLedger(l)
|
||||
_, _ = newServiceWithLedger(l).Suggest(context.Background(), rock, "en")
|
||||
if _, err := s.Report(context.Background(), 0); !errors.Is(err, ErrInvalidDays) {
|
||||
t.Errorf("0 days: %v", err)
|
||||
}
|
||||
if _, err := s.Report(context.Background(), MaxReportDays+1); !errors.Is(err, ErrInvalidDays) {
|
||||
t.Errorf("too many days: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newServiceWithLedger(l *fakeLedger) *Service {
|
||||
s := newService(fakeSettings{on: true, key: "sk-ant-key"}, &fakeClaude{res: anthropic.Result{Name: "N"}}, newStore())
|
||||
s.SetLedger(l)
|
||||
return s
|
||||
}
|
||||
|
||||
func TestAServiceWithNoLedgerGivesAnEmptyReport(t *testing.T) {
|
||||
s := newService(fakeSettings{}, &fakeClaude{}, newStore())
|
||||
r, err := s.Report(context.Background(), 30)
|
||||
if err != nil || len(r.Daily) != 0 {
|
||||
t.Errorf("%+v, %v", r, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
117
internal/naming/report.go
Normal file
117
internal/naming/report.go
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
package naming
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/providers/anthropic"
|
||||
)
|
||||
|
||||
// The longest period a report covers: the records are kept for a year.
|
||||
const MaxReportDays = 365
|
||||
|
||||
// ErrInvalidDays is what a report gives for a period that is not from 1 to MaxReportDays days.
|
||||
var ErrInvalidDays = errors.New("naming: days must be from 1 to 365")
|
||||
|
||||
// DayRow is what happened on one day (UTC) with one model.
|
||||
type DayRow struct {
|
||||
Day time.Time
|
||||
Model string
|
||||
Made int
|
||||
Kept int
|
||||
Failed int
|
||||
Usage anthropic.Usage
|
||||
}
|
||||
|
||||
// Ledger is the account of the calls: it records them and tells what happened since a moment.
|
||||
type Ledger interface {
|
||||
Meter
|
||||
// Since returns what happened from that moment on, a row for each day and model, the newest day first.
|
||||
Since(ctx context.Context, since time.Time) ([]DayRow, error)
|
||||
}
|
||||
|
||||
// SetLedger makes the service record every question it asks Claude and lets it report on them. Call it before the
|
||||
// service is used.
|
||||
func (s *Service) SetLedger(l Ledger) {
|
||||
s.meter = l
|
||||
s.ledger = l
|
||||
}
|
||||
|
||||
// Totals are the calls of a period, or of a day. CostUSD is an estimate worked out from the tokens and the price of the
|
||||
// model, not the invoice of Anthropic.
|
||||
type Totals struct {
|
||||
Made int // names Claude made
|
||||
Kept int // names taken from the kept ones, which cost nothing
|
||||
Failed int
|
||||
Usage anthropic.Usage
|
||||
CostUSD float64
|
||||
}
|
||||
|
||||
// Day is the totals of one day (UTC).
|
||||
type Day struct {
|
||||
Date string // 2006-01-02
|
||||
Totals
|
||||
}
|
||||
|
||||
// Report is what Claude costs over a period.
|
||||
type Report struct {
|
||||
Days int
|
||||
Model string // the model that is asked now
|
||||
Estimate bool // always true: the cost is worked out from the tokens
|
||||
Unpriced bool // some calls were to a model whose price is not known and are not in the cost
|
||||
Totals Totals
|
||||
Daily []Day // the days that had calls, the newest first
|
||||
}
|
||||
|
||||
// Summarize makes the report of the rows: the totals, a line for each day, and the cost of each in dollars.
|
||||
func Summarize(rows []DayRow, days int, model string) Report {
|
||||
r := Report{Days: days, Model: model, Estimate: true, Daily: []Day{}}
|
||||
byDay := map[string]*Day{}
|
||||
for _, row := range rows {
|
||||
date := row.Day.UTC().Format("2006-01-02")
|
||||
d := byDay[date]
|
||||
if d == nil {
|
||||
d = &Day{Date: date}
|
||||
byDay[date] = d
|
||||
}
|
||||
var cost float64
|
||||
if p, ok := anthropic.PriceOf(row.Model); ok {
|
||||
cost = p.Cost(row.Usage)
|
||||
} else if row.Usage != (anthropic.Usage{}) {
|
||||
r.Unpriced = true
|
||||
}
|
||||
for _, t := range []*Totals{&d.Totals, &r.Totals} {
|
||||
t.Made += row.Made
|
||||
t.Kept += row.Kept
|
||||
t.Failed += row.Failed
|
||||
t.Usage.Input += row.Usage.Input
|
||||
t.Usage.Output += row.Usage.Output
|
||||
t.Usage.CacheRead += row.Usage.CacheRead
|
||||
t.Usage.CacheWrite += row.Usage.CacheWrite
|
||||
t.CostUSD += cost
|
||||
}
|
||||
}
|
||||
for _, d := range byDay {
|
||||
r.Daily = append(r.Daily, *d)
|
||||
}
|
||||
sort.Slice(r.Daily, func(i, j int) bool { return r.Daily[i].Date > r.Daily[j].Date })
|
||||
return r
|
||||
}
|
||||
|
||||
// Report says what Claude cost over the last days (from 1 to MaxReportDays).
|
||||
func (s *Service) Report(ctx context.Context, days int) (Report, error) {
|
||||
if days < 1 || days > MaxReportDays {
|
||||
return Report{}, ErrInvalidDays
|
||||
}
|
||||
if s.ledger == nil {
|
||||
return Summarize(nil, days, anthropic.Model), nil
|
||||
}
|
||||
rows, err := s.ledger.Since(ctx, s.now().Add(-time.Duration(days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("read what Claude cost: %w", err)
|
||||
}
|
||||
return Summarize(rows, days, anthropic.Model), nil
|
||||
}
|
||||
|
|
@ -57,10 +57,41 @@ type Request struct {
|
|||
Language string
|
||||
}
|
||||
|
||||
// Result is a name and a line of description.
|
||||
// Usage is what a call used, in tokens, which is what it is billed by. Cache tokens are the ones read from and written
|
||||
// to the cache of Anthropic; Sonda sends nothing it asks to cache, so they are zero unless that changes.
|
||||
type Usage struct {
|
||||
Input, Output, CacheRead, CacheWrite int64
|
||||
}
|
||||
|
||||
// Result is a name and a line of description. Usage is set also when the call gave no name (ErrNoName), because
|
||||
// that answer was billed all the same.
|
||||
type Result struct {
|
||||
Name string
|
||||
Description string
|
||||
Usage Usage
|
||||
}
|
||||
|
||||
// Price is what a model costs, in dollars for each million tokens.
|
||||
type Price struct {
|
||||
Input, Output, CacheRead, CacheWrite float64
|
||||
}
|
||||
|
||||
// prices are the published prices of the models that Sonda asks. They are an estimate that the admin page says is one:
|
||||
// the invoice of Anthropic is the truth. A cache write is 1.25 times the price of an input token, as published.
|
||||
var prices = map[string]Price{
|
||||
"claude-opus-5-5": {Input: 4, Output: 20, CacheRead: 0.20, CacheWrite: 5},
|
||||
}
|
||||
|
||||
// PriceOf returns the price of a model, ok=false for one that is not known.
|
||||
func PriceOf(model string) (Price, bool) {
|
||||
p, ok := prices[model]
|
||||
return p, ok
|
||||
}
|
||||
|
||||
// Cost is what the usage costs, in dollars.
|
||||
func (p Price) Cost(u Usage) float64 {
|
||||
const million = 1_000_000
|
||||
return (float64(u.Input)*p.Input + float64(u.Output)*p.Output + float64(u.CacheRead)*p.CacheRead + float64(u.CacheWrite)*p.CacheWrite) / million
|
||||
}
|
||||
|
||||
// Client calls the Claude API. It is safe for concurrent use.
|
||||
|
|
@ -207,8 +238,12 @@ func (c *Client) Name(ctx context.Context, key string, req Request) (Result, err
|
|||
if err != nil {
|
||||
return Result{}, failure(err)
|
||||
}
|
||||
used := Usage{
|
||||
Input: resp.Usage.InputTokens, Output: resp.Usage.OutputTokens,
|
||||
CacheRead: resp.Usage.CacheReadInputTokens, CacheWrite: resp.Usage.CacheCreationInputTokens,
|
||||
}
|
||||
if resp.StopReason != sdk.StopReasonEndTurn {
|
||||
return Result{}, fmt.Errorf("%w: stopped with %s", ErrNoName, strconv.Quote(string(resp.StopReason)))
|
||||
return Result{Usage: used}, fmt.Errorf("%w: stopped with %s", ErrNoName, strconv.Quote(string(resp.StopReason)))
|
||||
}
|
||||
var text string
|
||||
for _, block := range resp.Content {
|
||||
|
|
@ -221,13 +256,13 @@ func (c *Client) Name(ctx context.Context, key string, req Request) (Result, err
|
|||
Description string `json:"description"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(text), &out); err != nil {
|
||||
return Result{}, fmt.Errorf("%w: not JSON", ErrNoName)
|
||||
return Result{Usage: used}, fmt.Errorf("%w: not JSON", ErrNoName)
|
||||
}
|
||||
name := truncate(strings.Trim(clean(out.Name), " \"'“”‘’«»"), MaxNameLen)
|
||||
if name == "" {
|
||||
return Result{}, fmt.Errorf("%w: empty", ErrNoName)
|
||||
return Result{Usage: used}, fmt.Errorf("%w: empty", ErrNoName)
|
||||
}
|
||||
return Result{Name: name, Description: truncate(clean(out.Description), MaxDescriptionLen)}, nil
|
||||
return Result{Name: name, Description: truncate(clean(out.Description), MaxDescriptionLen), Usage: used}, nil
|
||||
}
|
||||
|
||||
// clean takes out control characters and the blanks around and inside, so that a name is one line.
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ func message(stop, text string) string {
|
|||
b, _ := json.Marshal(map[string]any{
|
||||
"id": "msg_1", "type": "message", "role": "assistant", "model": Model, "stop_reason": stop,
|
||||
"content": []any{map[string]any{"type": "text", "text": text}},
|
||||
"usage": map[string]any{"input_tokens": 10, "output_tokens": 10},
|
||||
"usage": map[string]any{"input_tokens": 120, "output_tokens": 45, "cache_read_input_tokens": 7, "cache_creation_input_tokens": 3},
|
||||
})
|
||||
return string(b)
|
||||
}
|
||||
|
|
@ -213,3 +213,39 @@ func TestNothingIsTakenFromTheEnvironmentOfTheServer(t *testing.T) {
|
|||
t.Error("a header of the environment was sent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTokensThatWereBilledComeBackWithTheNameAndAlsoWithoutOne(t *testing.T) {
|
||||
want := Usage{Input: 120, Output: 45, CacheRead: 7, CacheWrite: 3}
|
||||
ok := newServer(t, 200, message("end_turn", `{"name":"x","description":"y"}`))
|
||||
got, err := ok.client().Name(context.Background(), "sk-ant-x", req)
|
||||
if err != nil || got.Usage != want {
|
||||
t.Errorf("a name: usage = %+v, err = %v, want %+v", got.Usage, err, want)
|
||||
}
|
||||
// A refusal and an answer that is no name still cost what they cost.
|
||||
for name, reply := range map[string]string{"refusal": message("refusal", ""), "cut": message("max_tokens", `{"name":"x"`), "not json": message("end_turn", "nope")} {
|
||||
s := newServer(t, 200, reply)
|
||||
res, err := s.client().Name(context.Background(), "sk-ant-x", req)
|
||||
if !errors.Is(err, ErrNoName) || res.Usage != want {
|
||||
t.Errorf("%s: usage = %+v, err = %v, want %+v with ErrNoName", name, res.Usage, err, want)
|
||||
}
|
||||
}
|
||||
// A call that did not get through has nothing to bill.
|
||||
bad := newServer(t, 500, `{"type":"error","error":{"type":"x","message":"y"}}`)
|
||||
if res, _ := bad.client().Name(context.Background(), "sk-ant-x", req); res.Usage != (Usage{}) {
|
||||
t.Errorf("a failed call: usage = %+v", res.Usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryModelThatIsUsedHasAPrice(t *testing.T) {
|
||||
p, ok := PriceOf(Model)
|
||||
if !ok || p.Input <= 0 || p.Output <= 0 {
|
||||
t.Fatalf("price of %s = %+v, %v", Model, p, ok)
|
||||
}
|
||||
// 1000 tokens read and 500 written at $4 and $20 per million.
|
||||
if got := p.Cost(Usage{Input: 1000, Output: 500}); got < 0.01399 || got > 0.01401 {
|
||||
t.Errorf("cost = %v, want 0.014", got)
|
||||
}
|
||||
if _, ok := PriceOf("claude-nobody"); ok {
|
||||
t.Error("a model that is not known has a price")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,3 +67,49 @@ func (r *NamingRepo) GenreCounts(ctx context.Context, trackIDs []int64) (map[str
|
|||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// keepUsage is how long the account of the calls is kept.
|
||||
const keepUsage = 365 * 24 * time.Hour
|
||||
|
||||
// Record keeps the account of one call to Claude, and forgets the ones that are older than a year.
|
||||
func (r *NamingRepo) Record(ctx context.Context, c naming.Call) error {
|
||||
_, err := r.db.ExecContext(ctx,
|
||||
`INSERT INTO claude_usage (at, kind, model, outcome, reason, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
c.At.Unix(), c.Kind, c.Model, c.Outcome, c.Reason, c.Usage.Input, c.Usage.Output, c.Usage.CacheRead, c.Usage.CacheWrite)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record a call: %w", err)
|
||||
}
|
||||
if _, err := r.db.ExecContext(ctx, "DELETE FROM claude_usage WHERE at < ?", c.At.Add(-keepUsage).Unix()); err != nil {
|
||||
return fmt.Errorf("forget old calls: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Since returns what happened from a moment on, a row for each day (UTC) and model, the newest day first.
|
||||
func (r *NamingRepo) Since(ctx context.Context, since time.Time) ([]naming.DayRow, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT at / 86400, model,
|
||||
SUM(CASE WHEN outcome = 'made' THEN 1 ELSE 0 END),
|
||||
SUM(CASE WHEN outcome = 'kept' THEN 1 ELSE 0 END),
|
||||
SUM(CASE WHEN outcome = 'failed' THEN 1 ELSE 0 END),
|
||||
SUM(input_tokens), SUM(output_tokens), SUM(cache_read_tokens), SUM(cache_write_tokens)
|
||||
FROM claude_usage WHERE at >= ? GROUP BY 1, 2 ORDER BY 1 DESC, 2`, since.Unix())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read what Claude cost: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []naming.DayRow{}
|
||||
for rows.Next() {
|
||||
var (
|
||||
d naming.DayRow
|
||||
day int64
|
||||
)
|
||||
if err := rows.Scan(&day, &d.Model, &d.Made, &d.Kept, &d.Failed, &d.Usage.Input, &d.Usage.Output, &d.Usage.CacheRead, &d.Usage.CacheWrite); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Day = time.Unix(day*86400, 0).UTC()
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/uptrace/bun"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/naming"
|
||||
"codeberg.org/squidd/sonda/internal/providers/anthropic"
|
||||
)
|
||||
|
||||
func TestNamingRepoKeepsAndReadsANameAndKeepsTheFirstOneWhenAskedTwice(t *testing.T) {
|
||||
|
|
@ -52,3 +53,74 @@ func TestNamingRepoCountsTheGenresOfSongs(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestUsageRepoSumsEachDayAndModelAndForgetsWhatIsOlderThanAYear(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
r := NewNamingRepo(db)
|
||||
day := func(n int) time.Time { return time.Date(2026, 11, n, 15, 30, 0, 0, time.UTC) }
|
||||
rec := func(at time.Time, kind, model, outcome, reason string, in, out int64) {
|
||||
t.Helper()
|
||||
err := r.Record(ctx, naming.Call{At: at, Kind: kind, Model: model, Outcome: outcome, Reason: reason, Usage: anthropic.Usage{Input: in, Output: out, CacheRead: 1, CacheWrite: 2}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rec(day(12), "mix", "claude-a", naming.OutcomeMade, "", 100, 40)
|
||||
rec(day(12), "smart", "claude-a", naming.OutcomeMade, "", 200, 60)
|
||||
rec(day(12), "mix", "claude-a", naming.OutcomeKept, "", 0, 0)
|
||||
rec(day(12), "mix", "claude-a", naming.OutcomeFailed, naming.ReasonNoName, 50, 300)
|
||||
rec(day(12), "mix", "claude-b", naming.OutcomeMade, "", 10, 10)
|
||||
rec(day(10), "playlist", "claude-a", naming.OutcomeFailed, naming.ReasonKey, 0, 0)
|
||||
rec(day(1), "mix", "claude-a", naming.OutcomeMade, "", 7, 7)
|
||||
|
||||
rows, err := r.Since(ctx, day(5))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 3 {
|
||||
t.Fatalf("rows = %+v, want 3: the 12th with two models and the 10th; the 1st is before", rows)
|
||||
}
|
||||
first := rows[0]
|
||||
if !first.Day.Equal(time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC)) || first.Model != "claude-a" || first.Made != 2 || first.Kept != 1 || first.Failed != 1 ||
|
||||
first.Usage != (anthropic.Usage{Input: 350, Output: 400, CacheRead: 4, CacheWrite: 8}) {
|
||||
t.Errorf("first = %+v", first)
|
||||
}
|
||||
if rows[1].Model != "claude-b" || rows[1].Made != 1 || rows[1].Usage.Input != 10 {
|
||||
t.Errorf("second = %+v", rows[1])
|
||||
}
|
||||
if !rows[2].Day.Equal(time.Date(2026, 11, 10, 0, 0, 0, 0, time.UTC)) || rows[2].Failed != 1 {
|
||||
t.Errorf("third = %+v", rows[2])
|
||||
}
|
||||
if none, _ := r.Since(ctx, day(30)); len(none) != 0 {
|
||||
t.Errorf("a period with no calls = %+v", none)
|
||||
}
|
||||
|
||||
// A call more than a year after the oldest one makes the oldest go.
|
||||
if err := r.Record(ctx, naming.Call{At: day(1).Add(366 * 24 * time.Hour), Kind: "mix", Model: "claude-a", Outcome: naming.OutcomeMade}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var n int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM claude_usage WHERE at < ?", day(1).Unix()+1).Scan(&n); err != nil || n != 0 {
|
||||
t.Errorf("%d records older than a year are left, %v", n, err)
|
||||
}
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM claude_usage").Scan(&n); err != nil || n == 0 {
|
||||
t.Errorf("the recent ones went too: %d, %v", n, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestUsageRepoKeepsNothingOfTheFacts(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
cols, err := db.Query("SELECT * FROM claude_usage LIMIT 0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cols.Close()
|
||||
names, _ := cols.Columns()
|
||||
want := []string{"id", "at", "kind", "model", "outcome", "reason", "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"}
|
||||
if len(names) != len(want) {
|
||||
t.Errorf("columns = %v, want %v: nothing but the account", names, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) {
|
|||
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if applied != 21 {
|
||||
t.Errorf("applied migrations = %d, want 21", applied)
|
||||
if applied != 22 {
|
||||
t.Errorf("applied migrations = %d, want 22", applied)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue