feat(import): keep the artist and the time of plays of artists I do not have

The plays of songs that are not in the library were only counted. Now, when
such a play counts as listened (not a skip, long enough), its artist and its
time are kept in a new table (migration 0031, outside_plays), so a model can
later learn which artists are played together, also those not in the library.
No title, album, IP address, country or platform is stored. The same play is
kept once however many times the export is imported, and the report says how
many were kept (history.kept_outside).
This commit is contained in:
tgiachi 2026-11-15 10:50:29 +01:00
commit 3ba087bcc5
9 changed files with 314 additions and 28 deletions

View file

@ -121,7 +121,7 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
feedbackSvc := feedback.NewService(storage.NewFeedbackRepo(db))
deps.Feedback = feedbackSvc
deps.Playlists = playlists.NewService(storage.NewPlaylistRepo(db))
deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.NewSpotifyImportRepo(db), storage.SearchKey, 20*time.Second)
deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.NewSpotifyImportRepo(db), storage.NewOutsidePlayRepo(db), storage.SearchKey, 20*time.Second)
deps.Mixes = mixes.NewService(storage.NewMixRepo(db), deps.Taste.(mixes.Taster), 20*time.Second)
deps.Smart = smart.NewService(storage.NewSmartRepo(db), deps.Taste.(smart.Taster), 20*time.Second)
namingSvc := naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil)

View file

@ -30,6 +30,7 @@ type spotifyHistoryJSON struct {
AlreadyThere int `json:"already_there"`
NotFound int `json:"not_found"`
Ignored int `json:"ignored"`
KeptOutside int `json:"kept_outside"`
}
type spotifyPlaylistJSON struct {
@ -61,7 +62,7 @@ func toSpotifyReport(run spotifyimport.Run) spotifyReportJSON {
ID: run.ID, CreatedAt: run.At.UTC(),
History: spotifyHistoryJSON{
Entries: r.History.Entries, Matched: r.History.Matched, Imported: r.History.Imported,
AlreadyThere: r.History.AlreadyThere, NotFound: r.History.NotFound, Ignored: r.History.Ignored,
AlreadyThere: r.History.AlreadyThere, NotFound: r.History.NotFound, Ignored: r.History.Ignored, KeptOutside: r.History.KeptOutside,
},
Playlists: make([]spotifyPlaylistJSON, 0, len(r.Playlists)),
}

View file

@ -82,8 +82,8 @@ func TestImportingAHistoryRecordsTheListensAndTheReportSaysWhatHappened(t *testi
t.Errorf("id and time = %v %v, want the import kept as 1", r.Body["id"], r.Body["created_at"])
}
hist, _ := r.Body["history"].(map[string]any)
sameKeys(t, "history", hist, "entries", "matched", "imported", "already_there", "not_found", "ignored")
if hist["entries"] != 2.0 || hist["matched"] != 1.0 || hist["imported"] != 1.0 || hist["not_found"] != 1.0 {
sameKeys(t, "history", hist, "entries", "matched", "imported", "already_there", "not_found", "ignored", "kept_outside")
if hist["entries"] != 2.0 || hist["matched"] != 1.0 || hist["imported"] != 1.0 || hist["not_found"] != 1.0 || hist["kept_outside"] != 1.0 {
t.Errorf("history = %v", hist)
}
missing, _ := r.Body["missing"].(map[string]any)
@ -248,3 +248,24 @@ func TestAnImportThatFindsNothingIsNotListed(t *testing.T) {
t.Errorf("without a token: %d, want 401", r.Code)
}
}
func TestThePlayOfAnArtistNotInTheLibraryKeepsOnlyTheArtistAndTheTime(t *testing.T) {
h, db := newTestEnv(t, func(*Deps) {})
seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead")
tok := registerAndLogin(t, h)
if r := upload(t, h, tok, spotifyHistory); r.Code != http.StatusOK {
t.Fatalf("import: %d %v", r.Code, r.Body)
}
var artist string
var at int64
if err := db.QueryRow("SELECT artist, played_at FROM outside_plays").Scan(&artist, &at); err != nil {
t.Fatal(err)
}
if artist != "Radiohead" || at != 1682935800 { // Creep, 2023-05-01T10:10:00Z, 100 s played
t.Errorf("kept %q at %d, want Radiohead at 1682935800", artist, at)
}
var columns int
if err := db.QueryRow("SELECT COUNT(*) FROM pragma_table_info('outside_plays')").Scan(&columns); err == nil && columns != 5 {
t.Errorf("outside_plays has %d columns, want the 5 that hold no title, album or address", columns)
}
}

View file

@ -2197,7 +2197,9 @@ paths:
Everything happens on this server and nothing is sent to anyone. The files are read from the request and not
kept, and of each play only the song, the artist, the album, the time and how it ended are read: the IP
addresses, the country and the platform in the file are never looked at.
addresses, the country and the platform in the file are never looked at. Of a play that is not in your
library and counts as listened, the name of the artist and the time are kept (and nothing else), so that
Sonda can learn which artists you play together.
A play is matched to a song of your library by artist and title (case, accents and punctuation do not matter,
a "Remastered" note on either side does not either, and any of the credited artists will do; a play of a
@ -5511,7 +5513,7 @@ components:
history:
type: object
additionalProperties: false
required: [entries, matched, imported, already_there, not_found, ignored]
required: [entries, matched, imported, already_there, not_found, ignored, kept_outside]
properties:
entries: {type: integer, description: 'Plays of songs that were read.'}
matched: {type: integer, description: 'Of those, the plays of a song of your library.'}
@ -5519,6 +5521,7 @@ components:
already_there: {type: integer, description: 'Matched and already recorded by an earlier import, or the same play in two files.'}
not_found: {type: integer, description: 'Plays of songs that are not in your library.'}
ignored: {type: integer, description: 'Podcast episodes, and plays with no time that can be read.'}
kept_outside: {type: integer, description: 'Of the plays not in your library, the ones that count as listened, whose artist and time are kept to learn what you listen to together (never the title, the album or anything else).'}
playlists:
type: array
items:

View file

@ -62,6 +62,19 @@ type Listens interface {
RecordMany(ctx context.Context, userID int64, events []listening.Event) (created int, err error)
}
// OutsidePlay is a play of an artist the person has no music of: the name and the time, nothing else.
type OutsidePlay struct {
Key string // the idempotency key of the play, the same as a listening event of it would have
Artist string
PlayedAt time.Time
}
// Outside keeps the plays of artists that are not in the library, so that what the person listens to can be learned
// from. A play whose key is already kept is not kept again, and is not counted in what comes back.
type Outside interface {
RecordOutside(ctx context.Context, userID int64, plays []OutsidePlay) (created int, err error)
}
// Playlists is what the import needs from the playlists of a person.
type Playlists interface {
List(ctx context.Context, userID int64, p playlists.Page) (playlists.Listing, error)
@ -83,6 +96,7 @@ type History struct {
AlreadyThere int `json:"already_there"` // matched, and already recorded by an earlier import (or the same play in two files)
NotFound int `json:"not_found"` // not in the library
Ignored int `json:"ignored"` // podcast episodes, and plays with no time that can be read
KeptOutside int `json:"kept_outside"` // of the not found, the plays that count as listened: the artist and the time are kept
}
// PlaylistResult says what became of one playlist. Status is `created`, `exists` (the person already has one with
@ -136,6 +150,7 @@ type Service struct {
listens Listens
lists Playlists
runs Runs
outside Outside
key func(string) string
minListen time.Duration
running sync.Map // user id -> struct{}
@ -143,8 +158,8 @@ type Service struct {
// NewService builds a Service. key is how names are made comparable (the key of the library search); minListen is how
// long a play must last to count as a listen, so that a play shorter than that is recorded as a skip.
func NewService(catalog Catalog, listens Listens, lists Playlists, runs Runs, key func(string) string, minListen time.Duration) *Service {
return &Service{catalog: catalog, listens: listens, lists: lists, runs: runs, key: key, minListen: minListen}
func NewService(catalog Catalog, listens Listens, lists Playlists, runs Runs, outside Outside, key func(string) string, minListen time.Duration) *Service {
return &Service{catalog: catalog, listens: listens, lists: lists, runs: runs, outside: outside, key: key, minListen: minListen}
}
// History lists the imports of a person that were kept, newest first. A limit of 0 is DefaultPage, and one above
@ -219,6 +234,8 @@ type run struct {
// memo remembers which song an artist, title and album are, so that a song played a thousand times is looked up once (0: not found).
memo map[string]int64
batch []listening.Event
away []OutsidePlay // plays of artists that are not in the library, waiting to be kept
kept int
matched int
created int
missing map[string]*Missing // by the comparable name of the artist
@ -256,6 +273,13 @@ func (r *run) entry(e Entry) {
if !ok {
r.report.History.NotFound++
r.miss(e.Artist)
// A skip says nothing about what goes together; a play that counts keeps its artist and its time.
if r.svc.kind(e, e.PlayedMs, 0) != listening.Skip {
r.away = append(r.away, OutsidePlay{Key: idempotencyKey(e), Artist: e.Artist, PlayedAt: e.PlayedAt})
if len(r.away) >= listenBatch {
r.err = r.flush()
}
}
return
}
r.matched++
@ -274,15 +298,21 @@ func (r *run) entry(e Entry) {
}
func (r *run) flush() error {
if len(r.batch) == 0 {
return nil
if len(r.batch) > 0 {
n, err := r.svc.listens.RecordMany(r.ctx, r.userID, r.batch)
if err != nil {
return fmt.Errorf("record the listens: %w", err)
}
r.created += n
r.batch = r.batch[:0]
}
n, err := r.svc.listens.RecordMany(r.ctx, r.userID, r.batch)
if err != nil {
return fmt.Errorf("record the listens: %w", err)
if len(r.away) > 0 {
if _, err := r.svc.outside.RecordOutside(r.ctx, r.userID, r.away); err != nil {
return fmt.Errorf("keep the plays of artists that are not in the library: %w", err)
}
r.kept += len(r.away)
r.away = r.away[:0]
}
r.created += n
r.batch = r.batch[:0]
return nil
}
@ -414,6 +444,7 @@ func (r *run) finish() Report {
h.Matched = r.matched
h.Imported = r.created
h.AlreadyThere = r.matched - r.created
h.KeptOutside = r.kept
all := make([]Missing, 0, len(r.missing))
for _, m := range r.missing {
all = append(all, *m)

View file

@ -126,16 +126,40 @@ func (f *fakeRuns) List(_ context.Context, _ int64, limit, offset int) ([]Run, i
return all[start:end], len(all), nil
}
// fakeOutside keeps the plays by key, as the database does.
type fakeOutside struct {
kept map[string]OutsidePlay
fail error
}
func (f *fakeOutside) RecordOutside(_ context.Context, _ int64, plays []OutsidePlay) (int, error) {
if f.fail != nil {
return 0, f.fail
}
if f.kept == nil {
f.kept = map[string]OutsidePlay{}
}
n := 0
for _, p := range plays {
if _, ok := f.kept[p.Key]; !ok {
f.kept[p.Key] = p
n++
}
}
return n, nil
}
type rig struct {
svc *Service
li *fakeListens
pl *fakePlaylists
runs *fakeRuns
out *fakeOutside
}
func newRig(cat *fakeCatalog) rig {
li, pl, runs := &fakeListens{}, &fakePlaylists{}, &fakeRuns{}
return rig{NewService(cat, li, pl, runs, key, 20*time.Second), li, pl, runs}
li, pl, runs, out := &fakeListens{}, &fakePlaylists{}, &fakeRuns{}, &fakeOutside{}
return rig{NewService(cat, li, pl, runs, out, key, 20*time.Second), li, pl, runs, out}
}
// report runs an import and gives the report of it.
@ -454,3 +478,58 @@ func TestHistoryPagesAreBounded(t *testing.T) {
t.Errorf("a negative offset: %v, want ErrInvalidPage", err)
}
}
func TestThePlaysOfArtistsNotInTheLibraryKeepOnlyTheArtistAndTheTime(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
rep, err := r.report(1, []File{upload(history(
play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 280_000, `,"reason_end":"trackdone"`), // in the library
play("2023-01-01T10:05:00Z", "Blur", "Song 2", "Blur", 120_000, `,"reason_end":"trackdone"`), // not, and it counts
play("2023-01-01T10:10:00Z", "Blur", "Parklife", "Parklife", 5_000, `,"skipped":true`), // not, and a skip
play("2023-01-01T10:15:00Z", "Oasis", "Wonderwall", "", 3_000, ``), // not, shorter than a listen
play("2023-01-01T10:20:00Z", "Oasis", "Supersonic", "", 90_000, ``), // not, played enough
))})
if err != nil {
t.Fatal(err)
}
if rep.History.NotFound != 4 || rep.History.KeptOutside != 2 {
t.Fatalf("history = %+v, want 4 not found and 2 kept", rep.History)
}
if len(r.out.kept) != 2 {
t.Fatalf("%d plays kept, want 2: %v", len(r.out.kept), r.out.kept)
}
got := map[string]time.Time{}
for k, p := range r.out.kept {
if !strings.HasPrefix(k, "spotify:") || p.Key != k {
t.Errorf("key %q of %+v", k, p)
}
got[p.Artist] = p.PlayedAt
}
if !got["Blur"].Equal(time.Date(2023, 1, 1, 10, 5, 0, 0, time.UTC)) || !got["Oasis"].Equal(time.Date(2023, 1, 1, 10, 20, 0, 0, time.UTC)) {
t.Errorf("kept = %v, want Blur at 10:05 and Oasis at 10:20", got)
}
}
func TestImportingAgainDoesNotKeepAPlayTwice(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
files := func() []File {
return []File{upload(history(play("2023-01-01T10:05:00Z", "Blur", "Song 2", "Blur", 120_000, `,"reason_end":"trackdone"`)))}
}
if _, err := r.report(1, files()); err != nil {
t.Fatal(err)
}
if _, err := r.report(1, files()); err != nil {
t.Fatal(err)
}
if len(r.out.kept) != 1 {
t.Errorf("%d plays kept after two imports, want 1", len(r.out.kept))
}
}
func TestAnErrorWhileKeepingThePlaysFailsTheImport(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
r.out.fail = errors.New("disk full")
_, err := r.report(1, []File{upload(history(play("2023-01-01T10:05:00Z", "Blur", "Song 2", "Blur", 120_000, `,"reason_end":"trackdone"`)))})
if err == nil || !strings.Contains(err.Error(), "disk full") {
t.Errorf("err = %v, want the disk error", err)
}
}

View file

@ -0,0 +1,60 @@
package storage
import (
"context"
"fmt"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/spotifyimport"
)
type outsidePlayRow struct {
bun.BaseModel `bun:"table:outside_plays"`
UserID int64 `bun:"user_id,pk"`
IdempotencyKey string `bun:"idempotency_key,pk"`
Artist string `bun:"artist,notnull"`
ArtistKey string `bun:"artist_key,notnull"`
PlayedAt int64 `bun:"played_at,notnull"`
}
// OutsidePlayRepo keeps the plays of artists a person has no music of. It implements spotifyimport.Outside.
type OutsidePlayRepo struct{ db *bun.DB }
// NewOutsidePlayRepo builds an OutsidePlayRepo.
func NewOutsidePlayRepo(db *bun.DB) *OutsidePlayRepo { return &OutsidePlayRepo{db: db} }
// outsideChunk is how many plays one statement inserts: five columns each.
const outsideChunk = 500
// RecordOutside keeps the plays of one person in one transaction and says how many were new. A play whose key is
// already there is skipped, also one that is twice in the batch.
func (r *OutsidePlayRepo) RecordOutside(ctx context.Context, userID int64, plays []spotifyimport.OutsidePlay) (int, error) {
created := 0
err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
for start := 0; start < len(plays); start += outsideChunk {
end := min(start+outsideChunk, len(plays))
rows := make([]outsidePlayRow, 0, end-start)
for _, p := range plays[start:end] {
rows = append(rows, outsidePlayRow{
UserID: userID, IdempotencyKey: p.Key, Artist: p.Artist, ArtistKey: SearchKey(p.Artist), PlayedAt: p.PlayedAt.Unix(),
})
}
res, err := tx.NewInsert().Model(&rows).On("CONFLICT (user_id, idempotency_key) DO NOTHING").Exec(ctx)
if err != nil {
return fmt.Errorf("insert outside plays: %w", err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("rows affected: %w", err)
}
created += int(n)
}
return nil
})
if err != nil {
return 0, err
}
return created, nil
}

View file

@ -177,3 +177,52 @@ func TestSpotifyImportsAreKeptPerUserNewestFirst(t *testing.T) {
}
})
}
func TestOutsidePlaysKeepOnlyTheArtistAndTheTimeAndAreKeptOnce(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
u, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false)
if err != nil {
t.Fatal(err)
}
repo := NewOutsidePlayRepo(db)
play := func(key, artist string, at int64) spotifyimport.OutsidePlay {
return spotifyimport.OutsidePlay{Key: key, Artist: artist, PlayedAt: time.Unix(at, 0)}
}
n, err := repo.RecordOutside(ctx, u.ID, []spotifyimport.OutsidePlay{play("a", "Björk", 100), play("b", "Blur", 200), play("a", "Björk", 100)})
if err != nil || n != 2 {
t.Fatalf("RecordOutside = %d, %v, want 2 new", n, err)
}
n, err = repo.RecordOutside(ctx, u.ID, []spotifyimport.OutsidePlay{play("a", "Björk", 100), play("c", "Oasis", 300)})
if err != nil || n != 1 {
t.Fatalf("second RecordOutside = %d, %v, want only the new one", n, err)
}
var artist, key string
if err := db.QueryRowContext(ctx, "SELECT artist, artist_key FROM outside_plays WHERE user_id = ? AND idempotency_key = 'a'", u.ID).Scan(&artist, &key); err != nil {
t.Fatal(err)
}
if artist != "Björk" || key != SearchKey("Björk") || key == "" {
t.Errorf("stored %q, %q, want the name and its search key", artist, key)
}
if got := count(t, db, "SELECT COUNT(*) FROM outside_plays WHERE user_id = ?", u.ID); got != 3 {
t.Errorf("%d plays stored, want 3", got)
}
if n, err := repo.RecordOutside(ctx, u.ID, nil); err != nil || n != 0 {
t.Errorf("an empty batch = %d, %v", n, err)
}
// More than one statement holds.
many := make([]spotifyimport.OutsidePlay, 1200)
for i := range many {
many[i] = play(fmt.Sprintf("k%d", i), "Many", int64(1000+i))
}
if n, err := repo.RecordOutside(ctx, u.ID, many); err != nil || n != 1200 {
t.Errorf("many = %d, %v, want 1200", n, err)
}
if _, err := db.ExecContext(ctx, "DELETE FROM users WHERE id = ?", u.ID); err != nil {
t.Fatal(err)
}
if got := count(t, db, "SELECT COUNT(*) FROM outside_plays"); got != 0 {
t.Errorf("%d plays left after the account is gone", got)
}
})
}

View file

@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) {
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil {
t.Fatal(err)
}
if applied != 30 {
t.Errorf("applied migrations = %d, want 30", applied)
if applied != 31 {
t.Errorf("applied migrations = %d, want 31", applied)
}
})
}