From 1857bb9c53413b20805fc541de1528998adf2d66 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 17:27:29 +0100 Subject: [PATCH 1/2] feat(discovery): remove provider data, genres from my files, full rescans - feat: DELETE /api/v1/providers/{name}/data (admin) removes everything a provider gave: its lists of similar artists, its tags and the records of when it was asked, then the artists that only that data named and that nobody has files for. Last.fm's terms ask for this when an agreement ends. The user's own artists, music, listens and settings stay, the provider is not switched off, and a second call removes nothing - feat: a discovered artist that no list names any more is removed when the list that named it is replaced, together with its tags, so they do not pile up. Real artists, and any artist that another list still names, are kept - feat: the scan reads the GENRE tag of each file, offline (several genres separated by ";", lower case, at most five per file), and GET /artists/{id}/tags gets a third entry, files: the genres of the artist's own playable tracks, each weighted by how many tracks have it. It asks nobody and is always there - feat: POST /library/scan and POST /libraries/{id}/scan take full=true to read every file again, not only the new or changed ones, so MusicBrainz ids and genres reach music that was indexed before. Files keep their identity, so listening history is kept. Anything but true or false is a 400 - test: the plan of a full scan, the scanner and the manager; the purge on both databases, checked by removing the rule that spares real artists; the cleanup of replaced lists; genres per file and per artist - docs: README, OpenAPI and the todo list updated --- README.md | 14 +- cmd/sondad/main.go | 3 + cmd/sondad/scan.go | 17 +- internal/api/http/api_test.go | 12 +- internal/api/http/contract_test.go | 37 +++- internal/api/http/handlers_library.go | 26 ++- internal/api/http/handlers_providers.go | 9 + internal/api/http/library_test.go | 41 ++++- internal/api/http/providers_test.go | 65 ++++++- internal/api/http/router.go | 6 +- internal/api/http/similar_test.go | 6 +- internal/api/http/tags_test.go | 33 +++- internal/api/openapi/openapi.yaml | 67 ++++++- internal/discovery/tags.go | 18 +- internal/discovery/tags_test.go | 40 +++- internal/library/manager.go | 16 +- internal/library/plan_test.go | 40 +++- internal/library/scanner.go | 32 +++- internal/library/scanner_test.go | 55 ++++++ internal/library/taglib.go | 27 +++ internal/library/taglib_test.go | 38 ++++ internal/library/types.go | 7 + internal/providers/settings.go | 29 +++ internal/providers/settings_test.go | 42 +++++ internal/storage/discovery.go | 94 ++++++++++ internal/storage/discovery_test.go | 212 +++++++++++++++++++++- internal/storage/library.go | 25 +++ internal/storage/storage_test.go | 6 +- migrations/postgres/0010_track_genres.sql | 9 + migrations/sqlite/0010_track_genres.sql | 9 + 30 files changed, 979 insertions(+), 56 deletions(-) create mode 100644 migrations/postgres/0010_track_genres.sql create mode 100644 migrations/sqlite/0010_track_genres.sql diff --git a/README.md b/README.md index fe75cdc..27c391e 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,12 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. -- **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights. See [External providers](#external-providers). +- **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights, and the genres of the artist's own files, read offline. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. ## What is not there yet -Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Genres from your own files, and the removal of provider data on request. +Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Recommendations from all this data (scoring) are next. ## Requirements @@ -200,7 +200,11 @@ The key is stored in the database, in clear text, and is never returned by the A **Tags.** `GET /api/v1/artists/{id}/tags` asks each provider that is on, at the same time, and keeps the answers with the same rules (7 days, 1 for an unknown artist, old data if a provider fails). Each provider has its own entry with its own weights: Last.fm's are relative to the artist (1 to 100), MusicBrainz's are counts of votes, and they are never mixed. MusicBrainz also marks the tags it lists as genres. Its tags and genres are CC BY-NC-SA, so show the `attribution` that comes with them. -**MusicBrainz ids come from your files, never from a guess.** If your files carry MusicBrainz artist ids in their tags (taggers like Picard write them), the scan reads them, offline, and keeps them on your artists, but only when each id can be paired with its artist for certain. MusicBrainz is asked only about artists whose id is known; for the others its entry says `no_id`. Sonda never searches MusicBrainz by name, because a search returns look-alikes and picking one would be a guess. The scan reads a file's tags when the file is new or changed, so to pick up ids for music that is already indexed, the files have to be touched or re-added. +**Genres from your own files.** The scan also reads the genre tag of each file (several genres separated by `;`, at most five per file) and `GET /artists/{id}/tags` adds a `files` entry: the genres of the artist's playable tracks, each weighted by how many tracks have it. It needs no provider, asks nobody, and is always there. + +**MusicBrainz ids come from your files, never from a guess.** If your files carry MusicBrainz artist ids in their tags (taggers like Picard write them), the scan reads them, offline, and keeps them on your artists, but only when each id can be paired with its artist for certain. MusicBrainz is asked only about artists whose id is known; for the others its entry says `no_id`. Sonda never searches MusicBrainz by name, because a search returns look-alikes and picking one would be a guess. A normal scan reads a file's tags when the file is new or changed. To pick up ids and genres for music that is already indexed, scan with `full=true` (`POST /library/scan?full=true`, or `POST /libraries/{id}/scan?full=true`): every file is read again, keeps its identity, and your listening history is untouched. + +**Taking it back.** `DELETE /api/v1/providers/{name}/data` (admin) removes everything a provider gave: its similar artists, its tags and the records of when it was asked, and then the artists that only that data named and that you have no files for. Last.fm's terms ask for this when an agreement ends. Your own artists, music and listens stay, and the provider is not switched off. An artist that no list names any more is also removed when the list that named it is replaced, so they do not pile up. The artist is sent to Last.fm by its MusicBrainz id when your files gave one (if Last.fm does not know the id, by name), otherwise by its name exactly as it is written in your library, and Sonda never lets Last.fm correct it into another artist. Names and ids from Last.fm are never written onto your own artists. @@ -224,8 +228,8 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er | signed in | `GET /listens` | Your own history, newest first, with `counted` per event. Query: `limit`, `offset`, `counted=true`. | | signed in | `GET /library/scan` | Scan status and the last result. | | admin | `GET`, `POST /libraries`; `PATCH`, `DELETE /libraries/{id}` | Manage libraries. Deleting one removes its index entries, never your files. | -| admin | `POST /library/scan`, `POST /libraries/{id}/scan` | Scan everything, or one library. | -| admin | `GET /providers`, `PUT /providers/{name}`, `POST /providers/{name}/test` | See, set up and check Last.fm and MusicBrainz. The API key is write-only. | +| admin | `POST /library/scan`, `POST /libraries/{id}/scan` | Scan everything, or one library. `?full=true` reads every file again. | +| admin | `GET /providers`, `PUT /providers/{name}`, `POST /providers/{name}/test`, `DELETE /providers/{name}/data` | See, set up and check Last.fm and MusicBrainz, and remove what they gave. The API key is write-only. | | admin | `GET`, `POST /users`; `PATCH`, `DELETE /users/{id}`; `PUT /users/{id}/password` | Manage users and the admin role. The last admin cannot be demoted or deleted. | A library path must be an absolute, existing folder, never the filesystem root, and never inside or around another library. Search is a case-insensitive substring match: `BJÖRK` finds `Björk`, but accents are kept, so `bjork` does not. diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 450d8a3..be6cd1b 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -128,6 +128,9 @@ func run() error { return musicbrainzClient.Probe(ctx) }) discoveryRepo := storage.NewDiscoveryRepo(db) + providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) { + return discoveryRepo.Purge(ctx, name) + }) discoveries := discovery.NewService(discoveryRepo, lastfmClient, providerSettings, logger) tags := discovery.NewTagService(discoveryRepo, map[string]discovery.TagSource{ providers.LastFM: discovery.LastfmTags{Client: lastfmClient}, diff --git a/cmd/sondad/scan.go b/cmd/sondad/scan.go index a26efbf..0679a67 100644 --- a/cmd/sondad/scan.go +++ b/cmd/sondad/scan.go @@ -15,9 +15,20 @@ type scanControl struct { m *library.Manager } -func (c scanControl) Start() error { return c.m.Start(c.ctx) } -func (c scanControl) StartLibrary(id int64) error { return c.m.StartLibrary(c.ctx, id) } -func (c scanControl) Status() library.Status { return c.m.Status() } +func (c scanControl) Start(full bool) error { return c.m.Start(c.ctx, scanOptions(full)...) } + +func (c scanControl) StartLibrary(id int64, full bool) error { + return c.m.StartLibrary(c.ctx, id, scanOptions(full)...) +} + +func (c scanControl) Status() library.Status { return c.m.Status() } + +func scanOptions(full bool) []library.ScanOption { + if full { + return []library.ScanOption{library.FullRescan} + } + return nil +} // logScanResults writes one log line per finished scan. File paths are left // out on purpose: audio paths are private. diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 89a6205..cfce655 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -75,7 +75,7 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun } tokens := auth.NewTokenIssuer(secret, time.Hour) - prov := providers.NewService(storage.NewProviderRepo(db)) + prov := newProviders(db) deps := Deps{ Logger: zap.NewNop(), Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, 24*time.Hour), @@ -92,6 +92,16 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun return NewRouter(deps), db } +// newProviders builds the provider service the way main does, with the real way +// of removing provider data. +func newProviders(db *bun.DB) *providers.Service { + svc := providers.NewService(storage.NewProviderRepo(db)) + svc.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) { + return storage.NewDiscoveryRepo(db).Purge(ctx, name) + }) + return svc +} + // noFiles is a streaming source with nothing to play. type noFiles struct{} diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index 532a536..81a9616 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -80,9 +80,22 @@ func (r *contractReader) Read(path string) (library.Metadata, error) { // scanAdapter lets the API start scans on a real manager. type scanAdapter struct{ m *library.Manager } -func (a scanAdapter) Start() error { return a.m.Start(context.Background()) } -func (a scanAdapter) StartLibrary(id int64) error { return a.m.StartLibrary(context.Background(), id) } -func (a scanAdapter) Status() library.Status { return a.m.Status() } +func (a scanAdapter) Start(full bool) error { + return a.m.Start(context.Background(), scanOptions(full)...) +} + +func (a scanAdapter) StartLibrary(id int64, full bool) error { + return a.m.StartLibrary(context.Background(), id, scanOptions(full)...) +} + +func (a scanAdapter) Status() library.Status { return a.m.Status() } + +func scanOptions(full bool) []library.ScanOption { + if full { + return []library.ScanOption{library.FullRescan} + } + return nil +} type contract struct { lastfm *fakeLastfm @@ -102,7 +115,7 @@ func newContract(t *testing.T) *contract { tagsMB := &fakeTags{tags: []discovery.Tag{{Name: "grunge", Weight: 12, Genre: true}}} lf := &fakeLastfm{result: []lastfm.Similar{{Name: "Contract Neighbour", MBID: "99999999-0000-0000-0000-000000000009", Match: 1}}} h, db := newTestEnvDB(t, func(d *Deps, db *bun.DB) { - prov := providers.NewService(storage.NewProviderRepo(db)) + prov := newProviders(db) prov.SetChecker(providers.LastFM, func(context.Context, string) error { return nil }) d.Providers = prov prov.SetChecker(providers.MusicBrainz, func(context.Context, string) error { return nil }) @@ -339,6 +352,7 @@ func TestResponsesMatchTheSpec(t *testing.T) { close(gate) c.scanDone(token) + c.expect(c.call("POST", "/library/scan?full=maybe", token, nil), 400, "scan with a bad parameter", "invalid_input") c.expect(c.call("POST", "/library/scan", token, nil), 202, "scan everything") c.scanDone(token) status := c.expect(c.call("GET", "/library/scan", token, nil), 200, "scan status") @@ -347,6 +361,12 @@ func TestResponsesMatchTheSpec(t *testing.T) { t.Fatalf("second scan = %v, want 3 files in 2 libraries, all unchanged", last) } + c.expect(c.call("POST", "/library/scan?full=true", token, nil), 202, "scan everything again, in full") + c.scanDone(token) + if full := c.expect(c.call("GET", "/library/scan", token, nil), 200, "status after a full scan")["last"].(map[string]any); full["updated"] != float64(3) || full["unchanged"] != float64(0) { + t.Fatalf("full scan = %v, want the 3 files read again", full) + } + c.expect(c.call("DELETE", "/libraries/"+extraID, token, nil), 204, "delete library") c.expect(c.call("DELETE", "/libraries/"+extraID, token, nil), 404, "delete library twice", "library_not_found") @@ -423,10 +443,15 @@ func TestResponsesMatchTheSpec(t *testing.T) { c.expect(c.call("GET", "/artists/"+artistID+"/similar?limit=abc", token, nil), 400, "similar with a bad limit", "invalid_input") c.expect(c.call("PUT", "/providers/musicbrainz", token, map[string]any{"enabled": true, "contact": "https://example.org/me"}), 200, "set up musicbrainz") tags := c.expect(c.call("GET", "/artists/"+artistID+"/tags", token, nil), 200, "tags")["sources"].([]any) - if len(tags) != 2 || tags[0].(map[string]any)["provider_status"] != "ok" || tags[1].(map[string]any)["provider_status"] != "no_id" { - t.Fatalf("tags = %v, want Last.fm ok and MusicBrainz without an id", tags) + if len(tags) != 3 || tags[0].(map[string]any)["provider_status"] != "ok" || tags[1].(map[string]any)["provider_status"] != "no_id" || tags[2].(map[string]any)["provider"] != "files" { + t.Fatalf("tags = %v, want Last.fm ok, MusicBrainz without an id and the files", tags) } c.expect(c.call("GET", "/artists/999/tags", token, nil), 404, "tags of an unknown artist", "artist_not_found") + purged := c.expect(c.call("DELETE", "/providers/lastfm/data", token, nil), 200, "purge a provider's data") + if purged["tags"] != float64(1) || purged["similar_artists"] != float64(1) { + t.Fatalf("purge = %v", purged) + } + c.expect(c.call("DELETE", "/providers/spotify/data", token, nil), 404, "purge an unknown provider", "provider_not_found") c.expect(c.call("GET", "/artists/abc/tags", token, nil), 400, "tags of a bad id", "invalid_input") // ---- playback ---- diff --git a/internal/api/http/handlers_library.go b/internal/api/http/handlers_library.go index c2645d4..0f95314 100644 --- a/internal/api/http/handlers_library.go +++ b/internal/api/http/handlers_library.go @@ -35,9 +35,27 @@ type scanStatusJSON struct { LastError string `json:"last_error,omitempty"` } +// fullParam reads ?full=. Only "true" and "false" are accepted, and it answers +// 400 itself otherwise: a typo must not quietly start a scan of another kind. +func fullParam(c *gin.Context) (full, ok bool) { + raw, present := c.GetQuery("full") + switch { + case !present, raw == "false": + return false, true + case raw == "true": + return true, true + } + writeError(c, http.StatusBadRequest, "invalid_input", "full must be true or false") + return false, false +} + // startScan launches a background scan of the configured roots. func (h *libraryHandlers) startScan(c *gin.Context) { - writeScanStarted(c, h.scan.Start()) + full, ok := fullParam(c) + if !ok { + return + } + writeScanStarted(c, h.scan.Start(full)) } // startLibraryScan launches a background scan of one library. @@ -46,7 +64,11 @@ func (h *libraryHandlers) startLibraryScan(c *gin.Context) { if !ok { return } - writeScanStarted(c, h.scan.StartLibrary(id)) + full, ok := fullParam(c) + if !ok { + return + } + writeScanStarted(c, h.scan.StartLibrary(id, full)) } func writeScanStarted(c *gin.Context, err error) { diff --git a/internal/api/http/handlers_providers.go b/internal/api/http/handlers_providers.go index a6c9364..45ba325 100644 --- a/internal/api/http/handlers_providers.go +++ b/internal/api/http/handlers_providers.go @@ -90,3 +90,12 @@ func (h *providersHandlers) test(c *gin.Context) { } c.JSON(http.StatusOK, gin.H{"ok": res.OK, "status": res.Status}) } + +func (h *providersHandlers) purge(c *gin.Context) { + res, err := h.svc.Purge(c.Request.Context(), c.Param("name")) + if err != nil { + writeProviderError(c, err) + return + } + c.JSON(http.StatusOK, gin.H{"similar_artists": res.SimilarArtists, "tags": res.Tags, "artists": res.Artists}) +} diff --git a/internal/api/http/library_test.go b/internal/api/http/library_test.go index 647ae95..4a056b5 100644 --- a/internal/api/http/library_test.go +++ b/internal/api/http/library_test.go @@ -13,10 +13,11 @@ type fakeScan struct { noLibs bool starts int lastOnly int64 // library of the last StartLibrary call + lastFull bool // whether the last start asked for a full rescan status library.Status } -func (f *fakeScan) Start() error { +func (f *fakeScan) Start(full bool) error { if f.noLibs { return library.ErrNoLibraries } @@ -25,10 +26,11 @@ func (f *fakeScan) Start() error { } f.running = true f.starts++ + f.lastFull = full return nil } -func (f *fakeScan) StartLibrary(id int64) error { +func (f *fakeScan) StartLibrary(id int64, full bool) error { if id == 999 { return library.ErrLibraryNotFound } @@ -38,6 +40,7 @@ func (f *fakeScan) StartLibrary(id int64) error { f.running = true f.starts++ f.lastOnly = id + f.lastFull = full return nil } @@ -96,3 +99,37 @@ func TestScanStatus(t *testing.T) { t.Errorf("last = %v", last) } } + +// A full rescan is asked for explicitly; anything else in the parameter is a mistake. +func TestScanCanBeAskedToReadEveryFileAgain(t *testing.T) { + scan := &fakeScan{} + h := newTestAPIWithScan(t, scan) + tok := registerAndLogin(t, h) + + for _, tt := range []struct { + query string + full bool + }{{"", false}, {"?full=false", false}, {"?full=true", true}} { + scan.running = false + if r := do(t, h, "POST", "/api/v1/library/scan"+tt.query, tok, nil); r.Code != http.StatusAccepted || scan.lastFull != tt.full { + t.Errorf("scan%s: %d full=%v, want %v", tt.query, r.Code, scan.lastFull, tt.full) + } + scan.running = false + if r := do(t, h, "POST", "/api/v1/libraries/7/scan"+tt.query, tok, nil); r.Code != http.StatusAccepted || scan.lastFull != tt.full || scan.lastOnly != 7 { + t.Errorf("library scan%s: %d full=%v only=%d", tt.query, r.Code, scan.lastFull, scan.lastOnly) + } + } + + starts := scan.starts + scan.running = false + for _, q := range []string{"?full=maybe", "?full=1", "?full=TRUE", "?full="} { + for _, path := range []string{"/api/v1/library/scan", "/api/v1/libraries/7/scan"} { + if r := do(t, h, "POST", path+q, tok, nil); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("%s%s: %d %v, want 400 invalid_input", path, q, r.Code, r.Body) + } + } + } + if scan.starts != starts { + t.Error("a scan started with a bad parameter") + } +} diff --git a/internal/api/http/providers_test.go b/internal/api/http/providers_test.go index e0d3722..24ae0f5 100644 --- a/internal/api/http/providers_test.go +++ b/internal/api/http/providers_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "net/http" + "strconv" "strings" "testing" @@ -13,7 +14,6 @@ import ( "go.uber.org/zap/zaptest/observer" "codeberg.org/squidd/sonda/internal/providers" - "codeberg.org/squidd/sonda/internal/storage" ) const lastfmKey = "0123456789abcdef0123456789abcdef" @@ -193,7 +193,7 @@ func TestProviderTest(t *testing.T) { var asked string var answer error h, _ := newTestEnvDB(t, func(d *Deps, db *bun.DB) { - prov := providers.NewService(storage.NewProviderRepo(db)) + prov := newProviders(db) prov.SetChecker(providers.LastFM, func(_ context.Context, key string) error { asked = key; return answer }) d.Providers = prov }) @@ -231,3 +231,64 @@ func TestProviderTest(t *testing.T) { t.Errorf("a provider without a test: %d %v", r.Code, r.Body) } } + +// ---- taking back a provider's data ---- + +func TestPurgeIsAdminOnly(t *testing.T) { + h := newTestAPI(t) + admin := registerAndLogin(t, h) + _, bob := addUser(t, h, admin, "bob", false) + if r := do(t, h, "DELETE", "/api/v1/providers/lastfm/data", "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("no token: %d", r.Code) + } + if r := do(t, h, "DELETE", "/api/v1/providers/lastfm/data", bob, nil); r.Code != http.StatusForbidden || errCode(r) != "admin_required" { + t.Errorf("normal user: %d %v", r.Code, r.Body) + } + if r := do(t, h, "DELETE", "/api/v1/providers/spotify/data", admin, nil); r.Code != http.StatusNotFound || errCode(r) != "provider_not_found" { + t.Errorf("unknown provider: %d %v", r.Code, r.Body) + } +} + +func TestPurgeRemovesWhatTheProviderGaveAndNothingOfMine(t *testing.T) { + e := newSimilarEnv(t) + e.enableLastfm(t) + + r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) // names Beta (mine) and Gamma (discovered) + gamma := int64(r.Body["items"].([]any)[1].(map[string]any)["artist"].(map[string]any)["id"].(float64)) + if a := do(t, e.h, "GET", "/api/v1/artists/"+strconv.FormatInt(gamma, 10), e.admin, nil); a.Code != http.StatusOK { + t.Fatalf("the discovered artist is not there to begin with: %d", a.Code) + } + + p := do(t, e.h, "DELETE", "/api/v1/providers/lastfm/data", e.admin, nil) + if p.Code != http.StatusOK { + t.Fatalf("%d %v", p.Code, p.Body) + } + sameKeys(t, "purge", p.Body, "similar_artists", "tags", "artists") + if p.Body["similar_artists"] != float64(2) || p.Body["tags"] != float64(0) || p.Body["artists"] != float64(1) { + t.Errorf("purge = %v, want 2 similar artists, no tags and the one artist Gamma", p.Body) + } + raw, _ := json.Marshal(p.Body) + if strings.Contains(string(raw), lastfmKey) { + t.Fatal("the answer contains the API key") + } + + if a := do(t, e.h, "GET", "/api/v1/artists/"+strconv.FormatInt(gamma, 10), e.admin, nil); a.Code != http.StatusNotFound { + t.Errorf("the discovered artist survived: %d", a.Code) + } + for _, id := range []int64{e.alpha, e.beta} { + if a := do(t, e.h, "GET", "/api/v1/artists/"+strconv.FormatInt(id, 10), e.admin, nil); a.Code != http.StatusOK { + t.Errorf("one of my artists was touched: %d", a.Code) + } + } + // The provider is as it was: still on, still with its key. + list := do(t, e.h, "GET", "/api/v1/providers", e.admin, nil).Body["providers"].([]any)[0].(map[string]any) + if list["enabled"] != true || list["has_api_key"] != true { + t.Errorf("the purge changed the provider: %v", list) + } + // And the next look asks again, since nothing is stored. + calls := e.source.count() + again := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) + if again.Code != http.StatusOK || e.source.count() != calls+1 { + t.Errorf("after the purge: %d, calls %d -> %d", again.Code, calls, e.source.count()) + } +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index 954db94..b3ec744 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -50,8 +50,8 @@ type streamService interface { } type scanService interface { - Start() error - StartLibrary(id int64) error + Start(full bool) error + StartLibrary(id int64, full bool) error Status() library.Status } @@ -59,6 +59,7 @@ type providerService interface { List(ctx context.Context) ([]providers.Info, error) Update(ctx context.Context, name string, p providers.Patch) (providers.Info, error) Test(ctx context.Context, name string) (providers.TestResult, error) + Purge(ctx context.Context, name string) (providers.Purged, error) } type tagService interface { @@ -160,6 +161,7 @@ func NewRouter(d Deps) *gin.Engine { admin.GET("/providers", prov.list) admin.PUT("/providers/:name", prov.update) admin.POST("/providers/:name/test", prov.test) + admin.DELETE("/providers/:name/data", prov.purge) u := &usersHandlers{svc: d.Auth} admin.GET("/users", u.list) admin.POST("/users", u.create) diff --git a/internal/api/http/similar_test.go b/internal/api/http/similar_test.go index b6a0858..732077e 100644 --- a/internal/api/http/similar_test.go +++ b/internal/api/http/similar_test.go @@ -80,7 +80,7 @@ func newSimilarEnvWith(t *testing.T, more func(*Deps, *bun.DB, *providers.Servic }} e := &similarEnv{source: src} e.h, e.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) { - prov := providers.NewService(storage.NewProviderRepo(db)) + prov := newProviders(db) d.Providers = prov d.Catalog = catalog.NewService(storage.NewCatalogRepo(db)) d.Discovery = discovery.NewService(storage.NewDiscoveryRepo(db), src, prov, zap.NewNop()) @@ -295,7 +295,7 @@ func TestWhatLeavesTheServerIsTheArtistAndTheKeyOnly(t *testing.T) { var alpha int64 h, db := newTestEnvDB(t, func(d *Deps, db *bun.DB) { - prov := providers.NewService(storage.NewProviderRepo(db)) + prov := newProviders(db) d.Providers = prov client := lastfm.New(outbound.New(outbound.Options{UserAgent: providers.UserAgent("1.2.3", "")}), remote.URL) d.Discovery = discovery.NewService(storage.NewDiscoveryRepo(db), client, prov, zap.NewNop()) @@ -373,7 +373,7 @@ func TestALeavingClientIsNotAServerError(t *testing.T) { core, logs := observer.New(zap.DebugLevel) src := &fakeLastfm{release: make(chan struct{}), result: []lastfm.Similar{{Name: "Gamma", Match: 1}}} h, db := newTestEnvDB(t, func(d *Deps, db *bun.DB) { - prov := providers.NewService(storage.NewProviderRepo(db)) + prov := newProviders(db) d.Providers = prov d.Logger = zap.New(core) d.Discovery = discovery.NewService(storage.NewDiscoveryRepo(db), src, prov, zap.NewNop()) diff --git a/internal/api/http/tags_test.go b/internal/api/http/tags_test.go index 1d2aaac..6d83805 100644 --- a/internal/api/http/tags_test.go +++ b/internal/api/http/tags_test.go @@ -91,8 +91,8 @@ func TestTagsWithEveryProviderOffAskNobody(t *testing.T) { } sameKeys(t, "tags", r.Body, "sources") sources := r.Body["sources"].([]any) - if len(sources) != 2 { - t.Fatalf("sources = %v", sources) + if len(sources) != 3 { + t.Fatalf("sources = %v, want Last.fm, MusicBrainz and the files", sources) } for i, name := range []string{"lastfm", "musicbrainz"} { s := sources[i].(map[string]any) @@ -188,7 +188,7 @@ func TestWhatLeavesTheServerForMusicBrainzIsTheIDAndTheContact(t *testing.T) { e := &similarEnv{} e.h, e.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) { - prov := providers.NewService(storage.NewProviderRepo(db)) + prov := newProviders(db) d.Providers = prov d.Catalog = catalog.NewService(storage.NewCatalogRepo(db)) mbClient := musicbrainz.New(outbound.New(outbound.Options{UserAgentFunc: prov.UserAgentFor(providers.MusicBrainz, "1.2.3")}), remote.URL+"/ws/2/") @@ -232,3 +232,30 @@ func TestWhatLeavesTheServerForMusicBrainzIsTheIDAndTheContact(t *testing.T) { } } } + +// The genres of my own files are always there, and come from nobody. +func TestTagsIncludeTheGenresOfMyOwnFiles(t *testing.T) { + e := newTagsEnv(t) + if _, err := e.db.Exec("INSERT INTO track_genres (track_id, genre) SELECT t.id, 'grunge' FROM tracks t WHERE t.title = 'Alpha song'"); err != nil { + t.Fatal(err) + } + if _, err := e.db.Exec("INSERT INTO track_genres (track_id, genre) SELECT t.id, 'rock' FROM tracks t WHERE t.title = 'Alpha song'"); err != nil { + t.Fatal(err) + } + r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) + files := r.Body["sources"].([]any)[2].(map[string]any) + sameKeys(t, "files", files, "provider", "provider_status", "stale", "retrieved_at", "attribution", "tags") + tags := files["tags"].([]any) + if files["provider"] != "files" || files["provider_status"] != "ok" || files["stale"] != false || len(tags) != 2 { + t.Fatalf("files = %v", files) + } + if first := tags[0].(map[string]any); first["name"] != "grunge" || first["weight"] != float64(1) || first["genre"] != true { + t.Errorf("first genre = %v", first) + } + if v, present := files["retrieved_at"]; !present || v != nil { + t.Errorf("retrieved_at = %#v, want an explicit null: nothing was read from anywhere", v) + } + if e.lf.calls+e.mb.calls != 0 { + t.Error("the genres of my files asked a provider") + } +} diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index e50d767..44dbdd7 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -458,8 +458,12 @@ paths: tags: [Scan] operationId: scanLibrary summary: Scan one library - description: Admin only. Starts a scan in the background; follow it with `GET /library/scan`. + description: | + Admin only. Starts a scan in the background; follow it with `GET /library/scan`. With + `full=true` every file is read again, not only the new or changed ones. security: [{bearerAuth: []}] + parameters: + - {$ref: '#/components/parameters/Full'} responses: '202': {$ref: '#/components/responses/ScanStarted'} '400': {$ref: '#/components/responses/InvalidInput'} @@ -494,9 +498,16 @@ paths: Admin only. Starts a scan in the background. Only MP3 and FLAC files are read; other audio formats are counted as `unsupported`. Housekeeping folders of NAS boxes (`@Recycle`, `#recycle`, `@eaDir`, `.@__thumb`, `$RECYCLE.BIN`) are skipped. + + A scan reads the tags of files that are new or changed. Send `full=true` to read every file + again: that is how MusicBrainz ids and genres reach music that was indexed before they were + read. Files keep their identity, so listening history is kept. security: [{bearerAuth: []}] + parameters: + - {$ref: '#/components/parameters/Full'} responses: '202': {$ref: '#/components/responses/ScanStarted'} + '400': {$ref: '#/components/responses/InvalidInput'} '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/AdminRequired'} '409': @@ -606,7 +617,10 @@ paths: summary: Tags of an artist, from each provider description: | What each provider says kind of music this artist plays: Last.fm's top tags and - MusicBrainz's tags and genres. There is always one entry for each, Last.fm first. The + MusicBrainz's tags and genres, and last the genres of the artist's own music files + (`provider` is `files`: the genre tags the scan read, each weighted by how many playable + tracks have it; it asks nobody, is always `ok` and has a null `retrieved_at`). There is + always one entry for each, Last.fm first. The weights are the providers' own numbers, kept as they came and never mixed: Last.fm's is relative to the artist (1 to 100, the top tag is usually 100; a tag with no votes is dropped), MusicBrainz's is a count of votes. `genre` is true for a MusicBrainz tag that it also lists among its curated genres. @@ -937,6 +951,39 @@ paths: application/json: schema: {$ref: '#/components/schemas/Error'} + /providers/{name}/data: + parameters: + - name: name + in: path + required: true + schema: {type: string, enum: [lastfm, musicbrainz]} + delete: + tags: [Providers] + operationId: purgeProviderData + summary: Remove what a provider gave + description: | + Admin only. Removes everything the provider told Sonda: its lists of similar artists, its + tags and the records of when it was asked, and then the artists that only that data named + and that nobody has files for. Last.fm's terms ask for exactly this when an agreement ends. + + Your own artists, music, listens and settings are not touched, and the provider is not + switched off: if it is still on, it is asked again when someone asks. The answer counts what + was removed; doing it again removes nothing. + security: [{bearerAuth: []}] + responses: + '200': + description: What was removed. + content: + application/json: + schema: {$ref: '#/components/schemas/ProviderPurge'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + '404': + description: No such provider (`provider_not_found`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + /listens: get: tags: [Playback] @@ -1035,6 +1082,11 @@ components: Case-insensitive substring of the name or title. Accented letters are matched ignoring case (`BJÖRK` finds `Björk`), but accents are kept (`bjork` does not). At most 200 characters. schema: {type: string, maxLength: 200} + Full: + name: full + in: query + description: "`true` reads every file again, not only the new or changed ones. Only `true` and `false` are accepted." + schema: {type: boolean, default: false} Limit: name: limit in: query @@ -1389,6 +1441,15 @@ components: maxLength: 200 description: A URL or e-mail address, without parentheses. Empty restores the default. + ProviderPurge: + type: object + additionalProperties: false + required: [similar_artists, tags, artists] + properties: + similar_artists: {type: integer, description: Similar-artist links removed.} + tags: {type: integer, description: Tags removed.} + artists: {type: integer, description: "Artists removed because only the removed data named them and nobody has files for them."} + ProviderTest: type: object additionalProperties: false @@ -1414,7 +1475,7 @@ components: additionalProperties: false required: [provider, provider_status, stale, retrieved_at, attribution, tags] properties: - provider: {type: string, enum: [lastfm, musicbrainz]} + provider: {type: string, enum: [lastfm, musicbrainz, files]} provider_status: type: string enum: [ok, disabled, error, no_id] diff --git a/internal/discovery/tags.go b/internal/discovery/tags.go index 4367e1a..2949053 100644 --- a/internal/discovery/tags.go +++ b/internal/discovery/tags.go @@ -25,6 +25,12 @@ const MaxTags = 50 // a page waiting for a minute because a client asked about fifty artists. const tagFetchTimeout = 20 * time.Second +// SourceFiles names the tags that come from the user's own music files. It is not +// a provider: it is always there, asks nobody and needs no settings. +const SourceFiles = "files" + +const attributionFiles = "From the genre tags of your own music files." + // StatusNoID says a provider that needs the MusicBrainz id of the artist cannot // be asked, because the id is not known. Sonda never looks one up by name. const StatusNoID = "no_id" @@ -62,13 +68,16 @@ type TagSourceResult struct { Tags []Tag } -// TagResult has one entry per provider, always in the same order. +// TagResult has one entry per provider, always in the same order, and the tags +// of the user's own files last. type TagResult struct{ Sources []TagSourceResult } // TagStore keeps the tags. Artist returns ErrNotFound for an id the catalog does // not know. type TagStore interface { Artist(ctx context.Context, id int64) (Artist, error) + // FileGenres counts the genres of the artist's own playable tracks, most first. + FileGenres(ctx context.Context, id int64) ([]Tag, error) Tags(ctx context.Context, id int64, provider string) (TagSnapshot, error) SaveTags(ctx context.Context, id int64, provider string, at time.Time, found bool, tags []Tag) error } @@ -134,12 +143,17 @@ func (s *TagService) Tags(ctx context.Context, artistID int64) (TagResult, error out[i], errs[i] = s.one(ctx, artist, p) }() } + genres, genErr := s.store.FileGenres(ctx, artistID) wg.Wait() - for _, err := range errs { + for _, err := range append(errs, genErr) { if err != nil { return TagResult{}, err } } + if genres == nil { + genres = []Tag{} + } + out = append(out, TagSourceResult{Provider: SourceFiles, Status: StatusOK, Attribution: attributionFiles, Tags: genres}) return TagResult{Sources: out}, nil } diff --git a/internal/discovery/tags_test.go b/internal/discovery/tags_test.go index 1f814a1..24543ee 100644 --- a/internal/discovery/tags_test.go +++ b/internal/discovery/tags_test.go @@ -26,6 +26,8 @@ type tagStore struct { artists map[int64]Artist snaps map[string]TagSnapshot saves []savedTags + genres map[int64][]Tag // what the artist's own files say + genErr error } func newTagStore() *tagStore { @@ -38,6 +40,12 @@ func newTagStore() *tagStore { } } +func (f *tagStore) FileGenres(_ context.Context, id int64) ([]Tag, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.genres[id], f.genErr +} + func tkey(id int64, provider string) string { return provider + "/" + string(rune('0'+id)) } func (f *tagStore) Artist(_ context.Context, id int64) (Artist, error) { @@ -149,10 +157,10 @@ func TestTagsOfAnUnknownArtistAreNotFoundAndNobodyIsAsked(t *testing.T) { func TestTagsAlwaysListBothProvidersInTheSameOrder(t *testing.T) { r := newTagRig(false, false) res := r.tags(t, 1) - if len(res.Sources) != 2 || res.Sources[0].Provider != providers.LastFM || res.Sources[1].Provider != providers.MusicBrainz { + if len(res.Sources) != 3 || res.Sources[0].Provider != providers.LastFM || res.Sources[1].Provider != providers.MusicBrainz || res.Sources[2].Provider != SourceFiles { t.Fatalf("sources = %+v", res.Sources) } - for _, s := range res.Sources { + for _, s := range res.Sources[:2] { if s.Status != StatusDisabled || len(s.Tags) != 0 || !s.RetrievedAt.IsZero() || s.Stale || s.Attribution == "" { t.Errorf("%s = %+v", s.Provider, s) } @@ -293,3 +301,31 @@ func TestConcurrentTagLooksShareOneCall(t *testing.T) { t.Errorf("calls = %d, want 1", r.lf.count()) } } + +// The genres of the user's own files need no provider: they are always there, +// ask nobody, and are computed when asked. +func TestTheGenresOfTheFilesAreAlwaysThere(t *testing.T) { + r := newTagRig(false, false) + r.store.genres = map[int64][]Tag{1: {{Name: "rock", Weight: 3, Genre: true}, {Name: "grunge", Weight: 1, Genre: true}}} + res := r.tags(t, 1) + files := res.Sources[2] + if files.Provider != SourceFiles || files.Status != StatusOK || files.Stale || len(files.Tags) != 2 || files.Tags[0].Name != "rock" || files.Tags[0].Weight != 3 || !files.RetrievedAt.IsZero() || files.Attribution == "" { + t.Errorf("files = %+v", files) + } + if r.lf.count()+r.mb.count() != 0 { + t.Error("reading the genres of the files asked a provider") + } + // An artist without files says nothing, and it is not an error. + res = r.tags(t, 2) + if f := res.Sources[2]; f.Status != StatusOK || f.Tags == nil || len(f.Tags) != 0 { + t.Errorf("files of an artist without any = %+v", f) + } +} + +func TestAFailureReadingFileGenresIsARealError(t *testing.T) { + r := newTagRig(false, false) + r.store.genErr = errors.New("db down") + if _, err := r.svc.Tags(context.Background(), 1); err == nil { + t.Fatal("the database failed and nobody was told") + } +} diff --git a/internal/library/manager.go b/internal/library/manager.go index 7c309da..559beb1 100644 --- a/internal/library/manager.go +++ b/internal/library/manager.go @@ -57,7 +57,7 @@ func (m *Manager) OnFinish(fn func(Summary, error)) { // Start scans every library in the background. ctx bounds the scan, so pass a // context that lives as long as the application, not an HTTP request. // It returns ErrNoLibraries when none are defined. -func (m *Manager) Start(ctx context.Context) error { +func (m *Manager) Start(ctx context.Context, opts ...ScanOption) error { libs, err := m.libraries.ListLibraries(ctx) if err != nil { return fmt.Errorf("list libraries: %w", err) @@ -65,20 +65,20 @@ func (m *Manager) Start(ctx context.Context) error { if len(libs) == 0 { return ErrNoLibraries } - return m.start(ctx, libs) + return m.start(ctx, libs, opts) } // StartLibrary scans a single library in the background. // It returns ErrLibraryNotFound for an unknown id. -func (m *Manager) StartLibrary(ctx context.Context, id int64) error { +func (m *Manager) StartLibrary(ctx context.Context, id int64, opts ...ScanOption) error { lib, err := m.libraries.GetLibrary(ctx, id) if err != nil { return err } - return m.start(ctx, []Library{lib}) + return m.start(ctx, []Library{lib}, opts) } -func (m *Manager) start(ctx context.Context, libs []Library) error { +func (m *Manager) start(ctx context.Context, libs []Library, opts []ScanOption) error { m.mu.Lock() defer m.mu.Unlock() if m.status.Running { @@ -86,12 +86,12 @@ func (m *Manager) start(ctx context.Context, libs []Library) error { } m.status.Running = true m.status.StartedAt = time.Now() - go m.run(ctx, libs) + go m.run(ctx, libs, opts) return nil } -func (m *Manager) run(ctx context.Context, libs []Library) { - sum, err := m.scanner.Scan(ctx, libs) +func (m *Manager) run(ctx context.Context, libs []Library, opts []ScanOption) { + sum, err := m.scanner.Scan(ctx, libs, opts...) m.mu.Lock() m.status.Running = false diff --git a/internal/library/plan_test.go b/internal/library/plan_test.go index 0af7096..f8db7e7 100644 --- a/internal/library/plan_test.go +++ b/internal/library/plan_test.go @@ -12,7 +12,7 @@ func TestPlanDoesNotFlagFilesWithStatErrors(t *testing.T) { files: []FileInfo{{RelPath: "new.mp3", Size: 10, MTimeUnix: 5}}, // same size+mtime as flaky.mp3 failedFiles: map[string]bool{"flaky.mp3": true}, } - p := makePlan(w, existing) + p := makePlan(w, existing, scanOptions{}) for _, id := range p.missing { if id == 1 { @@ -29,3 +29,41 @@ func TestPlanDoesNotFlagFilesWithStatErrors(t *testing.T) { t.Errorf("parse = %+v, want new.mp3 as a new file", p.parse) } } + +// A full rescan reads every file again, whether or not it changed, so what the +// reader learned since (ids, genres) reaches files that are already indexed. +func TestFullPlanReadsEveryPresentFileAgain(t *testing.T) { + existing := []ExistingFile{ + {ID: 1, FileInfo: FileInfo{RelPath: "same.mp3", Size: 10, MTimeUnix: 5}}, + {ID: 2, FileInfo: FileInfo{RelPath: "changed.mp3", Size: 20, MTimeUnix: 6}}, + {ID: 3, FileInfo: FileInfo{RelPath: "back.mp3", Size: 30, MTimeUnix: 7}, Missing: true}, + {ID: 4, FileInfo: FileInfo{RelPath: "gone.mp3", Size: 40, MTimeUnix: 8}}, + } + w := walkResult{files: []FileInfo{ + {RelPath: "same.mp3", Size: 10, MTimeUnix: 5}, + {RelPath: "changed.mp3", Size: 21, MTimeUnix: 6}, + {RelPath: "back.mp3", Size: 30, MTimeUnix: 7}, + {RelPath: "new.mp3", Size: 50, MTimeUnix: 9}, + }} + + normal := makePlan(w, existing, scanOptions{}) + if len(normal.parse) != 2 || normal.unchanged != 1 || len(normal.restore) != 1 { + t.Fatalf("normal plan: parse=%d unchanged=%d restore=%d", len(normal.parse), normal.unchanged, len(normal.restore)) + } + + full := makePlan(w, existing, scanOptions{full: true}) + ids := map[int64]bool{} + for _, j := range full.parse { + ids[j.existingID] = true + } + if len(full.parse) != 4 || !ids[1] || !ids[2] || !ids[3] || !ids[0] { + t.Errorf("full plan parses %+v, want the four present files (three known, one new)", full.parse) + } + if full.unchanged != 0 || len(full.restore) != 0 { + t.Errorf("full plan: unchanged=%d restore=%v, want none: a re-read clears the missing flag itself", full.unchanged, full.restore) + } + // What is gone is still gone. + if len(full.missing) != 1 || full.missing[0] != 4 { + t.Errorf("missing = %v, want only gone.mp3", full.missing) + } +} diff --git a/internal/library/scanner.go b/internal/library/scanner.go index aa6be5e..b480428 100644 --- a/internal/library/scanner.go +++ b/internal/library/scanner.go @@ -59,15 +59,35 @@ func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner { return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now} } +// ScanOption changes how a scan runs. +type ScanOption func(*scanOptions) + +type scanOptions struct{ full bool } + +// FullRescan makes a scan read the tags of every file again, not only of the files +// that are new or changed. It is how what the reader learned since (MusicBrainz +// ids, genres) reaches music that was indexed before. Nothing else changes: files +// keep their identity, so listening history is kept. +func FullRescan(o *scanOptions) { o.full = true } + +func buildOptions(opts []ScanOption) scanOptions { + var o scanOptions + for _, f := range opts { + f(&o) + } + return o +} + // Scan indexes every library in turn. An unreadable library is reported in the // returned error and the other libraries are still scanned. -func (s *Scanner) Scan(ctx context.Context, libs []Library) (Summary, error) { +func (s *Scanner) Scan(ctx context.Context, libs []Library, opts ...ScanOption) (Summary, error) { + o := buildOptions(opts) total := Summary{Libraries: len(libs)} s.publish(Event{Type: ScanStarted}) var errs []error for _, lib := range libs { - sum, err := s.scanLibrary(ctx, lib) + sum, err := s.scanLibrary(ctx, lib, o) total.add(sum) if err != nil { errs = append(errs, fmt.Errorf("%s: %w", lib.Name, err)) @@ -102,7 +122,7 @@ type parseResult struct { err error } -func (s *Scanner) scanLibrary(ctx context.Context, lib Library) (Summary, error) { +func (s *Scanner) scanLibrary(ctx context.Context, lib Library, o scanOptions) (Summary, error) { var sum Summary walked, err := s.walk(ctx, lib, &sum) @@ -126,7 +146,7 @@ func (s *Scanner) scanLibrary(ctx context.Context, lib Library) (Summary, error) "refusing to mark them missing", n) } } - p := makePlan(walked, existing) + p := makePlan(walked, existing, o) sum.Unchanged = p.unchanged if err := s.parseAndSave(ctx, lib, p.parse, &sum); err != nil { @@ -348,7 +368,7 @@ type fileKey struct{ size, mtime int64 } // A new file and a vanished file with the same size and modification time are // treated as a rename only when each is the sole candidate for that pair, so // look-alike files are never guessed. -func makePlan(w walkResult, existing []ExistingFile) plan { +func makePlan(w walkResult, existing []ExistingFile, o scanOptions) plan { var p plan byPath := make(map[string]ExistingFile, len(existing)) for _, e := range existing { @@ -365,7 +385,7 @@ func makePlan(w walkResult, existing []ExistingFile) plan { } seen[f.RelPath] = true switch { - case e.Size != f.Size || e.MTimeUnix != f.MTimeUnix: + case e.Size != f.Size || e.MTimeUnix != f.MTimeUnix || o.full: p.parse = append(p.parse, parseJob{file: f, existingID: e.ID}) case e.Missing: p.restore = append(p.restore, e.ID) diff --git a/internal/library/scanner_test.go b/internal/library/scanner_test.go index 9fe2976..24ab2ce 100644 --- a/internal/library/scanner_test.go +++ b/internal/library/scanner_test.go @@ -876,3 +876,58 @@ func TestScanCountsUnsupportedAudioFormats(t *testing.T) { t.Errorf("reader called %d times: unsupported files must not be opened", reader.calls.Load()) } } + +func TestFullScanReadsUnchangedFilesAgain(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + write(t, filepath.Join(root, "b.mp3"), "22") + + store, reader := newFakeStore(), &fakeReader{} + s, _ := newScanner(store, reader, 2) + mustScan(t, s, root) + idA := store.row(root, "a.mp3").ID + before := reader.calls.Load() + + sum, err := s.Scan(context.Background(), libsOf(root), FullRescan) + if err != nil { + t.Fatal(err) + } + if sum.Updated != 2 || sum.Unchanged != 0 || sum.Added != 0 { + t.Fatalf("full scan summary = %+v", sum) + } + if got := reader.calls.Load() - before; got != 2 { + t.Errorf("files read = %d, want 2", got) + } + if store.row(root, "a.mp3").ID != idA { + t.Error("a full scan changed the identity of a file") + } + // The next normal scan is back to skipping unchanged files. + before = reader.calls.Load() + if sum := mustScan(t, s, root); sum.Unchanged != 2 || reader.calls.Load() != before { + t.Errorf("after a full scan: %+v", sum) + } +} + +func TestManagerCanStartAFullScan(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + store, reader := newFakeStore(), &fakeReader{} + s, _ := newScanner(store, reader, 1) + mustScan(t, s, root) + before := reader.calls.Load() + + m := NewManager(s, fakeSource{libs: libsOf(root)}) + done := make(chan Summary, 1) + m.OnFinish(func(sum Summary, _ error) { done <- sum }) + if err := m.Start(context.Background(), FullRescan); err != nil { + t.Fatal(err) + } + select { + case sum := <-done: + if sum.Updated != 1 || reader.calls.Load() != before+1 { + t.Errorf("summary = %+v, reads = %d", sum, reader.calls.Load()-before) + } + case <-time.After(5 * time.Second): + t.Fatal("the scan did not finish") + } +} diff --git a/internal/library/taglib.go b/internal/library/taglib.go index 98dcee7..3b40695 100644 --- a/internal/library/taglib.go +++ b/internal/library/taglib.go @@ -6,6 +6,7 @@ import ( "regexp" "strconv" "strings" + "unicode/utf8" "go.senan.xyz/taglib" ) @@ -42,6 +43,7 @@ func (TaglibReader) Read(path string) (Metadata, error) { } m := Metadata{ Tags: Tags{ + Genres: cleanGenres(tags["GENRE"]), Title: first(tags, "TITLE"), Album: first(tags, "ALBUM"), AlbumArtist: albumArtist, @@ -73,6 +75,31 @@ func (TaglibReader) Read(path string) (Metadata, error) { return m, nil } +// maxGenreLen bounds one genre; a longer one is a sentence, not a genre. +const maxGenreLen = 100 + +// cleanGenres turns the values of the GENRE tag into a short list of lower case +// genres without repeats. A tagger may write several genres as several values or +// as one value joined with ";". A slash is not a separator: "R&B/Soul" is one genre. +func cleanGenres(values []string) []string { + var out []string + seen := map[string]bool{} + for _, v := range values { + for _, g := range strings.Split(v, ";") { + g = strings.ToLower(strings.TrimSpace(g)) + if g == "" || utf8.RuneCountInString(g) > maxGenreLen || seen[g] { + continue + } + seen[g] = true + out = append(out, g) + if len(out) == MaxGenresPerTrack { + return out + } + } + } + return out +} + var mbidRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) // splitIDs reads the MusicBrainz ids out of the values of one tag. Taggers write diff --git a/internal/library/taglib_test.go b/internal/library/taglib_test.go index 8f5aaf9..c9db592 100644 --- a/internal/library/taglib_test.go +++ b/internal/library/taglib_test.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" ) @@ -235,3 +236,40 @@ func TestTaglibReaderReadsMusicBrainzIDsFromMP3(t *testing.T) { t.Errorf("ids = %v: an ID3 'MusicBrainz Artist Id' frame should be read", m.ArtistMBIDs) } } + +// ---- genres ---- + +func TestGenresAreReadCleanedAndBounded(t *testing.T) { + p := makeAudio(t, t.TempDir(), "song.flac", map[string]string{ + "title": "T", "artist": "A", "genre": " Alternative Rock ; grunge;ALTERNATIVE ROCK; ; r&b/soul", + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + want := []string{"alternative rock", "grunge", "r&b/soul"} + if len(m.Genres) != len(want) { + t.Fatalf("genres = %v, want %v", m.Genres, want) + } + for i := range want { + if m.Genres[i] != want[i] { + t.Errorf("genres = %v, want %v", m.Genres, want) + } + } + + p = makeAudio(t, t.TempDir(), "none.flac", map[string]string{"title": "T", "artist": "A"}) + if m, _ = (TaglibReader{}).Read(p); len(m.Genres) != 0 { + t.Errorf("genres = %v, want none", m.Genres) + } +} + +func TestCleanGenres(t *testing.T) { + long := strings.Repeat("x", 101) + got := cleanGenres([]string{"a;b;c;d;e;f;g", long, "", " "}) + if len(got) != MaxGenresPerTrack || got[0] != "a" || got[MaxGenresPerTrack-1] != "e" { + t.Errorf("genres = %v, want the first %d", got, MaxGenresPerTrack) + } + if got := cleanGenres([]string{long}); len(got) != 0 { + t.Errorf("a 101-character genre is noise, kept as %v", got) + } +} diff --git a/internal/library/types.go b/internal/library/types.go index 13b1d6b..159d23e 100644 --- a/internal/library/types.go +++ b/internal/library/types.go @@ -17,7 +17,11 @@ var ErrUnreadable = errors.New("unreadable audio file") // The MusicBrainz ids are what the file itself says, read offline. ArtistMBIDs is // either empty or has one id per entry of Artists, in the same order: ids that // cannot be paired with their artists with certainty are not kept. +// +// Genres are the genre tags of the file, lower case, without repeats and at most +// MaxGenresPerTrack: a file tagged with twenty genres is not telling anything. type Tags struct { + Genres []string Title string Album string AlbumArtist string @@ -28,6 +32,9 @@ type Tags struct { DiscNo int } +// MaxGenresPerTrack bounds the genres kept for one file. +const MaxGenresPerTrack = 5 + // Properties are the technical properties of an audio file. type Properties struct { Format string // "mp3", "flac", ... diff --git a/internal/providers/settings.go b/internal/providers/settings.go index 5275dce..6233ce6 100644 --- a/internal/providers/settings.go +++ b/internal/providers/settings.go @@ -122,6 +122,7 @@ type Service struct { now func() time.Time mu sync.Mutex // an update reads then writes: keep two of them apart checkers map[string]Checker + purger Purger } // NewService builds a Service. @@ -354,3 +355,31 @@ func (s *Service) UserAgentFor(name, version string) func() string { return UserAgent(version, cur.Contact) } } + +// Purged says how much a purge removed. +type Purged struct { + SimilarArtists int // lists of similar artists + Tags int // tags + Artists int // artists that only the removed data named +} + +// Purger removes everything a provider gave, and reports how much. +type Purger func(ctx context.Context, name string) (Purged, error) + +// SetPurger registers how provider data is removed. Call it before the service is used. +func (s *Service) SetPurger(p Purger) { s.purger = p } + +// Purge removes everything a provider gave: what it said about artists, and the +// artists that only that data named and that nobody has files for. It works +// whether the provider is on or off and does not change that; a provider that is +// still on will be asked again when someone asks. Last.fm's terms ask for exactly +// this at the end of an agreement. The user's music, artists and listens stay. +func (s *Service) Purge(ctx context.Context, name string) (Purged, error) { + if _, ok := lookup(name); !ok { + return Purged{}, ErrUnknownProvider + } + if s.purger == nil { + return Purged{}, errors.New("no way to remove provider data was set up") + } + return s.purger(ctx, name) +} diff --git a/internal/providers/settings_test.go b/internal/providers/settings_test.go index 12d59fd..d58194e 100644 --- a/internal/providers/settings_test.go +++ b/internal/providers/settings_test.go @@ -366,3 +366,45 @@ func TestUserAgentForFollowsTheStoredContact(t *testing.T) { t.Errorf("with the store failing = %q, want the project address", got) } } + +// ---- purge ---- + +func TestPurgeRemovesAProvidersData(t *testing.T) { + s, _ := newSvc() + ctx := context.Background() + var asked string + s.SetPurger(func(_ context.Context, name string) (Purged, error) { + asked = name + return Purged{SimilarArtists: 3, Tags: 7, Artists: 2}, nil + }) + got, err := s.Purge(ctx, LastFM) + if err != nil || got != (Purged{SimilarArtists: 3, Tags: 7, Artists: 2}) || asked != LastFM { + t.Fatalf("purge = %+v err=%v asked=%q", got, err, asked) + } + // It works whether the provider is on or off, and does not change that. + if _, err := s.Update(ctx, LastFM, Patch{APIKey: strp(goodKey), Enabled: boolp(true)}); err != nil { + t.Fatal(err) + } + if _, err := s.Purge(ctx, LastFM); err != nil { + t.Fatal(err) + } + if _, ok, _ := s.Active(ctx, LastFM); !ok { + t.Error("a purge switched the provider off") + } +} + +func TestPurgeErrors(t *testing.T) { + s, _ := newSvc() + ctx := context.Background() + if _, err := s.Purge(ctx, "spotify"); !errors.Is(err, ErrUnknownProvider) { + t.Errorf("unknown provider: %v", err) + } + if _, err := s.Purge(ctx, LastFM); err == nil { + t.Error("a purge with nothing to purge with did not fail") + } + boom := errors.New("db down") + s.SetPurger(func(context.Context, string) (Purged, error) { return Purged{}, boom }) + if _, err := s.Purge(ctx, LastFM); !errors.Is(err, boom) { + t.Errorf("a storage error: %v", err) + } +} diff --git a/internal/storage/discovery.go b/internal/storage/discovery.go index b8dde5d..1aa3838 100644 --- a/internal/storage/discovery.go +++ b/internal/storage/discovery.go @@ -12,6 +12,7 @@ import ( "github.com/uptrace/bun/dialect" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/lastfm" ) @@ -126,6 +127,11 @@ func (r *DiscoveryRepo) SaveSimilar(ctx context.Context, id int64, provider stri if err != nil { return err } + // Remember who was named before, to remove the artists nobody names now. + var before []int64 + if err := tx.NewRaw("SELECT similar_id FROM artist_similar WHERE artist_id = ? AND provider = ?", id, provider).Scan(ctx, &before); err != nil { + return fmt.Errorf("read the old list: %w", err) + } if _, err := tx.ExecContext(ctx, "DELETE FROM artist_similar WHERE artist_id = ? AND provider = ?", id, provider); err != nil { return fmt.Errorf("clear similar artists: %w", err) } @@ -151,10 +157,31 @@ func (r *DiscoveryRepo) SaveSimilar(ctx context.Context, id int64, provider stri provider, kindSimilar, id, status, at.Unix()); err != nil { return fmt.Errorf("record fetch: %w", err) } + if len(before) > 0 { + if _, err := removeUnnamedArtists(ctx, tx, "id IN (?) AND ", bun.In(before)); err != nil { + return err + } + } return nil }) } +// removeUnnamedArtists deletes the discovered artists, among those matching the +// extra condition, that no list names and that name no list. Real artists (any +// that a file ever matched) are never touched. Their tags go with them. +func removeUnnamedArtists(ctx context.Context, tx bun.Tx, extra string, args ...any) (int, error) { + res, err := tx.ExecContext(ctx, + "DELETE FROM artists WHERE "+extra+`discovered = ? + AND NOT EXISTS (SELECT 1 FROM artist_similar s WHERE s.similar_id = artists.id) + AND NOT EXISTS (SELECT 1 FROM artist_similar s WHERE s.artist_id = artists.id)`, + append(args, true)...) + if err != nil { + return 0, fmt.Errorf("remove artists nobody names: %w", err) + } + n, _ := res.RowsAffected() + return int(n), nil +} + // resolveArtists returns the artist id of each candidate, creating the ones that // are not known. A candidate that is the artist itself, or an artist already // seen earlier in the list, gets 0. @@ -339,3 +366,70 @@ func (r *DiscoveryRepo) SaveTags(ctx context.Context, id int64, provider string, return nil }) } + +// MaxFileGenres bounds the genres returned for one artist. +const MaxFileGenres = 50 + +// FileGenres returns the genres of the artist's own playable tracks, each +// weighted by how many tracks have it, most first. It comes from the tags of the +// user's files and asks nobody. An artist without files has none. +func (r *DiscoveryRepo) FileGenres(ctx context.Context, artistID int64) ([]discovery.Tag, error) { + vis, visArgs := visible("t", 0) + name := "g.genre" + if r.db.Dialect().Name() == dialect.PG { + name += ` COLLATE "C"` // the same order as SQLite, whatever the database locale + } + rows, err := r.db.QueryContext(ctx, + `SELECT g.genre, COUNT(DISTINCT t.id) AS n + FROM track_genres g JOIN tracks t ON t.id = g.track_id JOIN track_artists ta ON ta.track_id = t.id + WHERE ta.artist_id = ? AND `+vis+` + GROUP BY g.genre ORDER BY n DESC, `+name+` LIMIT ?`, + append(append([]any{artistID}, visArgs...), MaxFileGenres)...) + if err != nil { + return nil, fmt.Errorf("read genres: %w", err) + } + defer rows.Close() + out := []discovery.Tag{} + for rows.Next() { + var ( + t discovery.Tag + n int + ) + if err := rows.Scan(&t.Name, &n); err != nil { + return nil, err + } + t.Weight, t.Genre = float64(n), true + out = append(out, t) + } + return out, rows.Err() +} + +// Purge removes everything a provider gave: its similar-artist lists, its tags and +// the records of when it was asked. Then it removes the artists that only that +// data named and that nobody has files for. The user's own artists, music and +// listening history are not touched. +func (r *DiscoveryRepo) Purge(ctx context.Context, provider string) (providers.Purged, error) { + var out providers.Purged + err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + for _, step := range []struct { + what string + query string + into *int + }{ + {"similar artists", "DELETE FROM artist_similar WHERE provider = ?", &out.SimilarArtists}, + {"tags", "DELETE FROM artist_tags WHERE provider = ?", &out.Tags}, + {"records", "DELETE FROM provider_fetches WHERE provider = ?", new(int)}, + } { + res, err := tx.ExecContext(ctx, step.query, provider) + if err != nil { + return fmt.Errorf("purge %s: %w", step.what, err) + } + n, _ := res.RowsAffected() + *step.into = int(n) + } + n, err := removeUnnamedArtists(ctx, tx, "") + out.Artists = n + return err + }) + return out, err +} diff --git a/internal/storage/discovery_test.go b/internal/storage/discovery_test.go index 2563058..fde9592 100644 --- a/internal/storage/discovery_test.go +++ b/internal/storage/discovery_test.go @@ -15,6 +15,7 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discovery" "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/lastfm" ) @@ -377,8 +378,11 @@ func TestConcurrentSavesDoNotFailOrDuplicate(t *testing.T) { if n := count(t, db, "SELECT COUNT(*) FROM (SELECT name_key FROM artists WHERE name LIKE 'Shared %' GROUP BY name_key HAVING COUNT(*) > 1) d"); n != 0 { t.Errorf("%d artists exist more than once", n) } - if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name LIKE 'Shared %'"); n != 2*rounds { - t.Errorf("shared artists = %d, want %d", n, 2*rounds) + // The lists are replaced every round, so the artists of earlier rounds that no + // list names are gone; the ones a scan adopted ("Shared With Id" of rounds + // 0 to rounds-2) are real artists now and stay, plus the two of the last round. + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name LIKE 'Shared %'"); n != rounds+1 { + t.Errorf("shared artists = %d, want %d", n, rounds+1) } // Each save replaces the source's edges, so what is left is the last round. if n := count(t, db, "SELECT COUNT(*) FROM artist_similar WHERE provider = 'lastfm'"); n != len(sources)*2 { @@ -727,3 +731,207 @@ func TestConfirmingTheIDKeepsWhatWasRead(t *testing.T) { } }) } + +// ---- genres from the files ---- + +func genresOf(t *testing.T, db *bun.DB, title string) []string { + t.Helper() + rows, err := db.Query("SELECT g.genre FROM track_genres g JOIN tracks t ON t.id = g.track_id WHERE t.title = ? ORDER BY g.genre", title) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []string + for rows.Next() { + var g string + if err := rows.Scan(&g); err != nil { + t.Fatal(err) + } + out = append(out, g) + } + return out +} + +func withGenres(f library.ParsedFile, genres ...string) library.ParsedFile { + f.Genres = genres + return f +} + +func TestScanKeepsTheGenresOfEachFile(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + lib := NewLibraryRepo(db) + + f := withGenres(pf("g/1.flac", "Genre Song", "", "", 0, 0, "flac", "Genre Artist"), "grunge", "rock") + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + if got := genresOf(t, db, "Genre Song"); !equal(got, []string{"grunge", "rock"}) { + t.Fatalf("genres = %v", got) + } + // Reading the file again replaces them, including with none. + var fileID int64 + _ = db.QueryRow("SELECT id FROM audio_files WHERE rel_path = 'g/1.flac'").Scan(&fileID) + f = withGenres(pf("g/1.flac", "Genre Song", "", "", 0, 0, "flac", "Genre Artist"), "metal") + if _, err := lib.SaveFile(ctx, s.rock, f, fileID); err != nil { + t.Fatal(err) + } + if got := genresOf(t, db, "Genre Song"); !equal(got, []string{"metal"}) { + t.Fatalf("after the update genres = %v", got) + } + f.Genres = nil + if _, err := lib.SaveFile(ctx, s.rock, f, fileID); err != nil { + t.Fatal(err) + } + if got := genresOf(t, db, "Genre Song"); len(got) != 0 { + t.Errorf("after clearing genres = %v", got) + } + }) +} + +func TestFileGenresCountTheArtistsPlayableTracks(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + lib, repo := NewLibraryRepo(db), NewDiscoveryRepo(db) + + save := func(rel, title string, artists []string, genres ...string) { + t.Helper() + f := withGenres(pf(rel, title, "", "", 0, 0, "flac", artists...), genres...) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + } + save("g/a.flac", "GA", []string{"Genre Band"}, "grunge", "rock") + save("g/b.flac", "GB", []string{"Genre Band"}, "rock") + save("g/c.flac", "GC", []string{"Genre Band", "Other Band"}, "rock", "post-grunge") + save("g/d.flac", "GD", []string{"Other Band"}, "jazz") + save("g/e.flac", "GE", []string{"Genre Band"}, "metal") // its file will be missing + if _, err := db.Exec("UPDATE audio_files SET missing_at = 1 WHERE rel_path = 'g/e.flac'"); err != nil { + t.Fatal(err) + } + + got, err := repo.FileGenres(ctx, artistID(t, db, "Genre Band")) + if err != nil { + t.Fatal(err) + } + want := []discovery.Tag{{Name: "rock", Weight: 3, Genre: true}, {Name: "grunge", Weight: 1, Genre: true}, {Name: "post-grunge", Weight: 1, Genre: true}} + if len(got) != len(want) { + t.Fatalf("genres = %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("genre %d = %+v, want %+v", i, got[i], want[i]) + } + } + // An artist nobody has files for has none. + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}}) + if got, err := repo.FileGenres(ctx, artistID(t, db, "Elbow")); err != nil || len(got) != 0 { + t.Errorf("a discovered artist: %+v err=%v", got, err) + } + }) +} + +// ---- taking back what a provider said ---- + +func TestPurgeRemovesWhatOneProviderGaveAndWhatOnlyItNamed(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + radiohead, bjork := artistID(t, db, "Radiohead"), artistID(t, db, "Björk") + + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}, {Name: "Muse", Match: 0.5}, {Name: "Björk", Match: 0.2}}) + _ = repo.SaveSimilar(ctx, bjork, "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}}) + elbow := artistID(t, db, "Elbow") + _ = repo.SaveTags(ctx, radiohead, "lastfm", at, true, []discovery.Tag{{Name: "rock", Weight: 100}}) + _ = repo.SaveTags(ctx, elbow, "lastfm", at, true, []discovery.Tag{{Name: "indie", Weight: 100}}) + _ = repo.SaveTags(ctx, radiohead, "musicbrainz", at, true, []discovery.Tag{{Name: "art rock", Weight: 30}, {Name: "rock", Weight: 19}}) + + // Purging MusicBrainz takes its tags and its records, and nothing else. + got, err := repo.Purge(ctx, "musicbrainz") + if err != nil { + t.Fatal(err) + } + if got.Tags != 2 || got.SimilarArtists != 0 || got.Artists != 0 { + t.Errorf("musicbrainz purge = %+v", got) + } + if tg, _ := repo.Tags(ctx, radiohead, "musicbrainz"); tg.Fetched || len(tg.Tags) != 0 { + t.Errorf("musicbrainz tags are still there: %+v", tg) + } + if tg, _ := repo.Tags(ctx, radiohead, "lastfm"); !tg.Fetched || len(tg.Tags) != 1 { + t.Errorf("lastfm tags were touched: %+v", tg) + } + + // Purging Last.fm takes everything it gave, and the artists only it named. + got, err = repo.Purge(ctx, "lastfm") + if err != nil { + t.Fatal(err) + } + if got.SimilarArtists != 4 || got.Tags != 2 || got.Artists != 2 { + t.Errorf("lastfm purge = %+v, want 4 similar artists, 2 tags and the 2 artists Elbow and Muse", got) + } + for _, table := range []string{"artist_similar", "artist_tags", "provider_fetches"} { + if n := count(t, db, "SELECT COUNT(*) FROM "+table); n != 0 { + t.Errorf("%s has %d rows left", table, n) + } + } + for _, name := range []string{"Elbow", "Muse"} { + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name = '"+name+"'"); n != 0 { + t.Errorf("the discovered artist %s is still there", name) + } + } + // The user's own artists, tracks and files are as they were. + for _, name := range []string{"Radiohead", "Björk", "Miles Davis"} { + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name = '"+name+"'"); n != 1 { + t.Errorf("%s: %d rows, want 1", name, n) + } + } + if n := count(t, db, "SELECT COUNT(*) FROM audio_files"); n == 0 { + t.Error("the purge touched the files") + } + // Doing it again finds nothing. + if again, err := repo.Purge(ctx, "lastfm"); err != nil || again != (providers.Purged{}) { + t.Errorf("second purge = %+v err=%v", again, err) + } + }) +} + +// A discovered artist that nothing points to any more is not kept: otherwise +// every list that changes leaves some behind. +func TestReplacingAListRemovesTheArtistsNoOneNamesAnymore(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + radiohead, bjork := artistID(t, db, "Radiohead"), artistID(t, db, "Björk") + + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{ + {Name: "Dropped", Match: 1}, {Name: "Shared", Match: 0.8}, {Name: "Kept", Match: 0.6}, {Name: "Miles Davis", Match: 0.4}}) + _ = repo.SaveSimilar(ctx, bjork, "lastfm", at, true, []lastfm.Similar{{Name: "Shared", Match: 1}}) + _ = repo.SaveTags(ctx, artistID(t, db, "Dropped"), "lastfm", at, true, []discovery.Tag{{Name: "x", Weight: 1}}) + + if err := repo.SaveSimilar(ctx, radiohead, "lastfm", at.Add(time.Hour), true, []lastfm.Similar{{Name: "Kept", Match: 1}}); err != nil { + t.Fatal(err) + } + has := func(name string) bool { + return count(t, db, "SELECT COUNT(*) FROM artists WHERE name = '"+name+"'") == 1 + } + if has("Dropped") { + t.Error("Dropped is no longer named by anyone but is still there") + } + if !has("Shared") { + t.Error("Shared is still named by Björk's list and was removed") + } + if !has("Kept") { + t.Error("Kept is still in the list and was removed") + } + if !has("Miles Davis") { + t.Error("a real artist was removed because a list stopped naming it") + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_tags"); n != 0 { + t.Errorf("tags of the removed artist left: %d", n) + } + }) +} diff --git a/internal/storage/library.go b/internal/storage/library.go index c9b6516..406472f 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -156,6 +156,9 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P return fmt.Errorf("insert audio file: %w", err) } added = true + if err := replaceGenres(ctx, tx, track.ID, f.Genres); err != nil { + return err + } return replaceCredits(ctx, tx, track.ID, artistIDs) } @@ -177,6 +180,9 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P WherePK().Exec(ctx); err != nil { return fmt.Errorf("update audio file: %w", err) } + if err := replaceGenres(ctx, tx, track.ID, f.Genres); err != nil { + return err + } return replaceCredits(ctx, tx, track.ID, artistIDs) }) if err != nil { @@ -499,6 +505,25 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6 return found.ID, nil } +// replaceGenres makes the genres of a track those the file says now. +func replaceGenres(ctx context.Context, tx bun.Tx, trackID int64, genres []string) error { + if _, err := tx.ExecContext(ctx, "DELETE FROM track_genres WHERE track_id = ?", trackID); err != nil { + return fmt.Errorf("clear genres: %w", err) + } + seen := map[string]bool{} + for _, g := range genres { + g = strings.ToLower(strings.TrimSpace(g)) + if g == "" || seen[g] || len(seen) == library.MaxGenresPerTrack { + continue + } + seen[g] = true + if _, err := tx.ExecContext(ctx, "INSERT INTO track_genres (track_id, genre) VALUES (?, ?)", trackID, g); err != nil { + return fmt.Errorf("insert genre: %w", err) + } + } + return nil +} + func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) { var a albumModel err := tx.NewSelect().Model(&a). diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 6212e82..04cd17a 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -147,7 +147,7 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := Migrate(context.Background(), db, fsys); err != nil { t.Fatalf("second Migrate: %v", err) } - for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches", "artist_tags"} { + for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches", "artist_tags", "track_genres"} { if !tableExists(t, db, driver, table) { t.Errorf("table %s missing", table) } @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 9 { - t.Errorf("applied migrations = %d, want 9", applied) + if applied != 10 { + t.Errorf("applied migrations = %d, want 10", applied) } }) } diff --git a/migrations/postgres/0010_track_genres.sql b/migrations/postgres/0010_track_genres.sql new file mode 100644 index 0000000..f58a2c6 --- /dev/null +++ b/migrations/postgres/0010_track_genres.sql @@ -0,0 +1,9 @@ +-- The genre tags of the music files, read offline by the scanner. A track has a +-- handful at most (the scanner keeps five), lower case. They are the user's own +-- data: they come from the files and are never sent anywhere. + +CREATE TABLE track_genres ( + track_id BIGINT NOT NULL REFERENCES tracks(id) ON DELETE CASCADE, + genre TEXT NOT NULL, + PRIMARY KEY (track_id, genre) +); diff --git a/migrations/sqlite/0010_track_genres.sql b/migrations/sqlite/0010_track_genres.sql new file mode 100644 index 0000000..64bfec3 --- /dev/null +++ b/migrations/sqlite/0010_track_genres.sql @@ -0,0 +1,9 @@ +-- The genre tags of the music files, read offline by the scanner. A track has a +-- handful at most (the scanner keeps five), lower case. They are the user's own +-- data: they come from the files and are never sent anywhere. + +CREATE TABLE track_genres ( + track_id INTEGER NOT NULL REFERENCES tracks(id) ON DELETE CASCADE, + genre TEXT NOT NULL, + PRIMARY KEY (track_id, genre) +); From b0a7a562a655c7452a51160f59d8a06f39a16d8c Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 17:36:14 +0100 Subject: [PATCH 2/2] fix(discovery): purge lock, cleanup chains and rescans that cannot read - fix: on PostgreSQL a purge could delete a discovered artist while a scan was adopting it. The scan then failed on the missing artist for the whole file, and the two could also deadlock. Purge now takes the same artist lock as the scanner and discovery before it removes anything. Saving a list or tags does too, and an artist that is gone by then is reported as not found instead of a foreign key error, so a user who opened an artist that was removed meanwhile gets a 404 - fix: a discovered artist that was only the source of a list stayed forever and kept the artists of its own list alive, so chains built up. Cleanup now removes such an artist and then follows the chain until nothing more goes - fix: in a full rescan a file that came back but could not be read stayed marked missing. It is restored first, as in a normal scan, and read as well - fix: ?full given twice is refused, so a typo cannot start a scan of another kind - fix: genre cleaning lives in one place, used by the reader and the store - docs: the files genres count the tracks the artist is credited on, and the spec says every files tag is a genre - test: a purge colliding with a scan on both databases (it fails without the lock), a chain of discovered artists, saving for an artist that is gone, a file that returns unreadable during a full scan --- README.md | 2 +- internal/api/http/handlers_library.go | 6 +- internal/api/http/library_test.go | 2 +- internal/api/openapi/openapi.yaml | 7 +- internal/discovery/discovery.go | 3 + internal/discovery/discovery_test.go | 14 ++++ internal/discovery/tags.go | 3 + internal/discovery/tags_test.go | 12 ++++ internal/library/plan_test.go | 6 +- internal/library/scanner.go | 7 +- internal/library/scanner_test.go | 33 ++++++++++ internal/library/taglib.go | 6 +- internal/library/taglib_test.go | 4 +- internal/storage/discovery.go | 70 ++++++++++++++++---- internal/storage/discovery_test.go | 93 +++++++++++++++++++++++++++ internal/storage/library.go | 11 +--- 16 files changed, 243 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 27c391e..572fbf1 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ The key is stored in the database, in clear text, and is never returned by the A **Tags.** `GET /api/v1/artists/{id}/tags` asks each provider that is on, at the same time, and keeps the answers with the same rules (7 days, 1 for an unknown artist, old data if a provider fails). Each provider has its own entry with its own weights: Last.fm's are relative to the artist (1 to 100), MusicBrainz's are counts of votes, and they are never mixed. MusicBrainz also marks the tags it lists as genres. Its tags and genres are CC BY-NC-SA, so show the `attribution` that comes with them. -**Genres from your own files.** The scan also reads the genre tag of each file (several genres separated by `;`, at most five per file) and `GET /artists/{id}/tags` adds a `files` entry: the genres of the artist's playable tracks, each weighted by how many tracks have it. It needs no provider, asks nobody, and is always there. +**Genres from your own files.** The scan also reads the genre tag of each file (several genres separated by `;`, at most five per file) and `GET /artists/{id}/tags` adds a `files` entry: the genres of the playable tracks the artist is credited on, each weighted by how many tracks have it. It needs no provider, asks nobody, and is always there. **MusicBrainz ids come from your files, never from a guess.** If your files carry MusicBrainz artist ids in their tags (taggers like Picard write them), the scan reads them, offline, and keeps them on your artists, but only when each id can be paired with its artist for certain. MusicBrainz is asked only about artists whose id is known; for the others its entry says `no_id`. Sonda never searches MusicBrainz by name, because a search returns look-alikes and picking one would be a guess. A normal scan reads a file's tags when the file is new or changed. To pick up ids and genres for music that is already indexed, scan with `full=true` (`POST /library/scan?full=true`, or `POST /libraries/{id}/scan?full=true`): every file is read again, keeps its identity, and your listening history is untouched. diff --git a/internal/api/http/handlers_library.go b/internal/api/http/handlers_library.go index 0f95314..f4fe083 100644 --- a/internal/api/http/handlers_library.go +++ b/internal/api/http/handlers_library.go @@ -38,11 +38,11 @@ type scanStatusJSON struct { // fullParam reads ?full=. Only "true" and "false" are accepted, and it answers // 400 itself otherwise: a typo must not quietly start a scan of another kind. func fullParam(c *gin.Context) (full, ok bool) { - raw, present := c.GetQuery("full") + values := c.QueryArray("full") switch { - case !present, raw == "false": + case len(values) == 0, len(values) == 1 && values[0] == "false": return false, true - case raw == "true": + case len(values) == 1 && values[0] == "true": return true, true } writeError(c, http.StatusBadRequest, "invalid_input", "full must be true or false") diff --git a/internal/api/http/library_test.go b/internal/api/http/library_test.go index 4a056b5..4a48112 100644 --- a/internal/api/http/library_test.go +++ b/internal/api/http/library_test.go @@ -122,7 +122,7 @@ func TestScanCanBeAskedToReadEveryFileAgain(t *testing.T) { starts := scan.starts scan.running = false - for _, q := range []string{"?full=maybe", "?full=1", "?full=TRUE", "?full="} { + for _, q := range []string{"?full=maybe", "?full=1", "?full=TRUE", "?full=", "?full=true&full=bogus", "?full=false&full=true"} { for _, path := range []string{"/api/v1/library/scan", "/api/v1/libraries/7/scan"} { if r := do(t, h, "POST", path+q, tok, nil); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { t.Errorf("%s%s: %d %v, want 400 invalid_input", path, q, r.Code, r.Body) diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 44dbdd7..81b2617 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -618,12 +618,13 @@ paths: description: | What each provider says kind of music this artist plays: Last.fm's top tags and MusicBrainz's tags and genres, and last the genres of the artist's own music files - (`provider` is `files`: the genre tags the scan read, each weighted by how many playable - tracks have it; it asks nobody, is always `ok` and has a null `retrieved_at`). There is + (`provider` is `files`: the genre tags the scan read, each weighted by how many of the + playable tracks the artist is credited on have it; it asks nobody, is always `ok` and has a null `retrieved_at`). There is always one entry for each, Last.fm first. The weights are the providers' own numbers, kept as they came and never mixed: Last.fm's is relative to the artist (1 to 100, the top tag is usually 100; a tag with no votes is dropped), - MusicBrainz's is a count of votes. `genre` is true for a MusicBrainz tag that it also lists among its curated genres. + MusicBrainz's is a count of votes. `genre` is true for a MusicBrainz tag that it also lists among its curated genres, and for + every tag of the `files` entry, which are genres by definition. A provider is only asked when an admin switched it on and its stored answer is missing or old (7 days; a "no such artist" answer is kept 1 day), with the same rules as diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index 388e995..d321443 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -200,6 +200,9 @@ func (s *Service) Similar(ctx context.Context, artistID int64, limit int) (Resul case <-ctx.Done(): return Result{}, ctx.Err() case r := <-done: + if errors.Is(r.Err, ErrNotFound) { + return Result{}, ErrNotFound // removed while it was being read + } if r.Err != nil { s.log.Warn("similar artists could not be read", zap.String("provider", Provider), zap.Error(r.Err)) return result(snap, StatusError, snap.Fetched), nil diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go index 39166a7..5e05244 100644 --- a/internal/discovery/discovery_test.go +++ b/internal/discovery/discovery_test.go @@ -33,6 +33,7 @@ type fakeStore struct { snaps map[int64]Snapshot saves []saved lastLim int + saveErr error // returned by SaveSimilar when set } func newStore() *fakeStore { @@ -66,6 +67,9 @@ func (f *fakeStore) Similar(_ context.Context, id int64, _ string, limit int) (S func (f *fakeStore) SaveSimilar(_ context.Context, id int64, provider string, at time.Time, found bool, cands []lastfm.Similar) error { f.mu.Lock() defer f.mu.Unlock() + if f.saveErr != nil { + return f.saveErr + } f.saves = append(f.saves, saved{id, provider, at, found, cands}) snap := Snapshot{Fetched: true, NotFound: !found, RetrievedAt: at} for i, c := range cands { @@ -413,3 +417,13 @@ func TestNoPageCanAskForMoreThanIsStored(t *testing.T) { t.Errorf("MaxLimit = %d is above FetchLimit = %d", MaxLimit, FetchLimit) } } + +// An artist removed while its list was being read is not there any more: that is +// "not found", not a provider failure. +func TestAnArtistRemovedWhileItsListWasReadIsNotFound(t *testing.T) { + r := newRig(true) + r.store.saveErr = ErrNotFound + if _, err := r.svc.Similar(context.Background(), 1, 10); !errors.Is(err, ErrNotFound) { + t.Fatalf("err = %v, want ErrNotFound", err) + } +} diff --git a/internal/discovery/tags.go b/internal/discovery/tags.go index 2949053..b74afcf 100644 --- a/internal/discovery/tags.go +++ b/internal/discovery/tags.go @@ -199,6 +199,9 @@ func (s *TagService) one(ctx context.Context, a Artist, p tagProvider) (TagSourc case <-ctx.Done(): return TagSourceResult{}, ctx.Err() case r := <-done: + if errors.Is(r.Err, ErrNotFound) { + return TagSourceResult{}, ErrNotFound // removed while it was being read + } if r.Err != nil { s.log.Warn("tags could not be read", zap.String("provider", p.name), zap.Error(r.Err)) return res(StatusError, snap.Fetched, snap), nil diff --git a/internal/discovery/tags_test.go b/internal/discovery/tags_test.go index 24543ee..e7bc667 100644 --- a/internal/discovery/tags_test.go +++ b/internal/discovery/tags_test.go @@ -28,6 +28,7 @@ type tagStore struct { saves []savedTags genres map[int64][]Tag // what the artist's own files say genErr error + saveErr error // returned by SaveTags when set } func newTagStore() *tagStore { @@ -67,6 +68,9 @@ func (f *tagStore) Tags(_ context.Context, id int64, provider string) (TagSnapsh func (f *tagStore) SaveTags(_ context.Context, id int64, provider string, at time.Time, found bool, tags []Tag) error { f.mu.Lock() defer f.mu.Unlock() + if f.saveErr != nil { + return f.saveErr + } f.saves = append(f.saves, savedTags{id, provider, at, found, tags}) f.snaps[tkey(id, provider)] = TagSnapshot{Fetched: true, NotFound: !found, RetrievedAt: at, Tags: tags} return nil @@ -329,3 +333,11 @@ func TestAFailureReadingFileGenresIsARealError(t *testing.T) { t.Fatal("the database failed and nobody was told") } } + +func TestAnArtistRemovedWhileItsTagsWereReadIsNotFound(t *testing.T) { + r := newTagRig(true, false) + r.store.saveErr = ErrNotFound + if _, err := r.svc.Tags(context.Background(), 1); !errors.Is(err, ErrNotFound) { + t.Fatalf("err = %v, want ErrNotFound", err) + } +} diff --git a/internal/library/plan_test.go b/internal/library/plan_test.go index f8db7e7..77ac561 100644 --- a/internal/library/plan_test.go +++ b/internal/library/plan_test.go @@ -59,8 +59,10 @@ func TestFullPlanReadsEveryPresentFileAgain(t *testing.T) { if len(full.parse) != 4 || !ids[1] || !ids[2] || !ids[3] || !ids[0] { t.Errorf("full plan parses %+v, want the four present files (three known, one new)", full.parse) } - if full.unchanged != 0 || len(full.restore) != 0 { - t.Errorf("full plan: unchanged=%d restore=%v, want none: a re-read clears the missing flag itself", full.unchanged, full.restore) + // A file that came back is restored even if reading it fails this time, as in a + // normal scan, and it is read as well. + if full.unchanged != 0 || len(full.restore) != 1 || full.restore[0] != 3 { + t.Errorf("full plan: unchanged=%d restore=%v, want only back.mp3 restored", full.unchanged, full.restore) } // What is gone is still gone. if len(full.missing) != 1 || full.missing[0] != 4 { diff --git a/internal/library/scanner.go b/internal/library/scanner.go index b480428..e61bdd4 100644 --- a/internal/library/scanner.go +++ b/internal/library/scanner.go @@ -384,9 +384,14 @@ func makePlan(w walkResult, existing []ExistingFile, o scanOptions) plan { continue } seen[f.RelPath] = true + changed := e.Size != f.Size || e.MTimeUnix != f.MTimeUnix switch { - case e.Size != f.Size || e.MTimeUnix != f.MTimeUnix || o.full: + case changed || o.full: p.parse = append(p.parse, parseJob{file: f, existingID: e.ID}) + if e.Missing && !changed { + // Back, and read again. It is back even if reading fails this time. + p.restore = append(p.restore, e.ID) + } case e.Missing: p.restore = append(p.restore, e.ID) default: diff --git a/internal/library/scanner_test.go b/internal/library/scanner_test.go index 24ab2ce..ed7facb 100644 --- a/internal/library/scanner_test.go +++ b/internal/library/scanner_test.go @@ -931,3 +931,36 @@ func TestManagerCanStartAFullScan(t *testing.T) { t.Fatal("the scan did not finish") } } + +// A file that comes back and cannot be read during a full scan is still back. +func TestFullScanStillRestoresAFileItCannotRead(t *testing.T) { + root := t.TempDir() + p := filepath.Join(root, "a.mp3") + write(t, p, "1") + store, reader := newFakeStore(), &fakeReader{} + s, _ := newScanner(store, reader, 1) + mustScan(t, s, root) + + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + write(t, filepath.Join(root, "other.mp3"), "22") // so the root is not empty + mustScan(t, s, root) + if !store.row(root, "a.mp3").Missing { + t.Fatal("the file should be missing") + } + + write(t, p, "1") // same size; the mtime is set below to what was indexed + old := store.row(root, "a.mp3").MTimeUnix + if err := os.Chtimes(p, time.Unix(old, 0), time.Unix(old, 0)); err != nil { + t.Fatal(err) + } + reader.fail = map[string]bool{"a.mp3": true} + sum, err := s.Scan(context.Background(), libsOf(root), FullRescan) + if err != nil { + t.Fatal(err) + } + if sum.Restored != 1 || sum.Failed != 1 || store.row(root, "a.mp3").Missing { + t.Errorf("summary = %+v, missing = %v: the file is there, even if its tags could not be read", sum, store.row(root, "a.mp3").Missing) + } +} diff --git a/internal/library/taglib.go b/internal/library/taglib.go index 3b40695..c5c9e74 100644 --- a/internal/library/taglib.go +++ b/internal/library/taglib.go @@ -43,7 +43,7 @@ func (TaglibReader) Read(path string) (Metadata, error) { } m := Metadata{ Tags: Tags{ - Genres: cleanGenres(tags["GENRE"]), + Genres: CleanGenres(tags["GENRE"]), Title: first(tags, "TITLE"), Album: first(tags, "ALBUM"), AlbumArtist: albumArtist, @@ -78,10 +78,10 @@ func (TaglibReader) Read(path string) (Metadata, error) { // maxGenreLen bounds one genre; a longer one is a sentence, not a genre. const maxGenreLen = 100 -// cleanGenres turns the values of the GENRE tag into a short list of lower case +// CleanGenres turns the values of the GENRE tag into a short list of lower case // genres without repeats. A tagger may write several genres as several values or // as one value joined with ";". A slash is not a separator: "R&B/Soul" is one genre. -func cleanGenres(values []string) []string { +func CleanGenres(values []string) []string { var out []string seen := map[string]bool{} for _, v := range values { diff --git a/internal/library/taglib_test.go b/internal/library/taglib_test.go index c9db592..dc37302 100644 --- a/internal/library/taglib_test.go +++ b/internal/library/taglib_test.go @@ -265,11 +265,11 @@ func TestGenresAreReadCleanedAndBounded(t *testing.T) { func TestCleanGenres(t *testing.T) { long := strings.Repeat("x", 101) - got := cleanGenres([]string{"a;b;c;d;e;f;g", long, "", " "}) + got := CleanGenres([]string{"a;b;c;d;e;f;g", long, "", " "}) if len(got) != MaxGenresPerTrack || got[0] != "a" || got[MaxGenresPerTrack-1] != "e" { t.Errorf("genres = %v, want the first %d", got, MaxGenresPerTrack) } - if got := cleanGenres([]string{long}); len(got) != 0 { + if got := CleanGenres([]string{long}); len(got) != 0 { t.Errorf("a 101-character genre is noise, kept as %v", got) } } diff --git a/internal/storage/discovery.go b/internal/storage/discovery.go index 1aa3838..a14ea01 100644 --- a/internal/storage/discovery.go +++ b/internal/storage/discovery.go @@ -123,6 +123,9 @@ func (r *DiscoveryRepo) SaveSimilar(ctx context.Context, id int64, provider stri if err := lockArtistWrites(ctx, tx); err != nil { return err } + if err := requireArtist(ctx, tx, id); err != nil { + return err + } ids, err := resolveArtists(ctx, tx, id, cands) if err != nil { return err @@ -166,22 +169,39 @@ func (r *DiscoveryRepo) SaveSimilar(ctx context.Context, id int64, provider stri }) } -// removeUnnamedArtists deletes the discovered artists, among those matching the -// extra condition, that no list names and that name no list. Real artists (any -// that a file ever matched) are never touched. Their tags go with them. +// removeUnnamedArtists deletes the discovered artists that no list names, among +// those matching the extra condition first, and then, over and over until nothing +// more goes, among all of them: removing an artist removes the list it was the +// source of, which can leave the artists only that list named without anyone +// naming them. Real artists (any that a file ever matched) are never touched. +// Tags and records of a removed artist go with it. The caller holds the artist +// lock, so nothing is adopted or named while this runs. func removeUnnamedArtists(ctx context.Context, tx bun.Tx, extra string, args ...any) (int, error) { - res, err := tx.ExecContext(ctx, - "DELETE FROM artists WHERE "+extra+`discovered = ? - AND NOT EXISTS (SELECT 1 FROM artist_similar s WHERE s.similar_id = artists.id) - AND NOT EXISTS (SELECT 1 FROM artist_similar s WHERE s.artist_id = artists.id)`, - append(args, true)...) - if err != nil { - return 0, fmt.Errorf("remove artists nobody names: %w", err) + total := 0 + for pass := 0; pass < maxCleanupPasses; pass++ { + cond, condArgs := "", []any(nil) + if pass == 0 { + cond, condArgs = extra, args + } + res, err := tx.ExecContext(ctx, + "DELETE FROM artists WHERE "+cond+`discovered = ? + AND NOT EXISTS (SELECT 1 FROM artist_similar s WHERE s.similar_id = artists.id)`, + append(condArgs, true)...) + if err != nil { + return total, fmt.Errorf("remove artists nobody names: %w", err) + } + n, _ := res.RowsAffected() + total += int(n) + if n == 0 { + break // nothing went, so nothing is left unnamed by it + } } - n, _ := res.RowsAffected() - return int(n), nil + return total, nil } +// maxCleanupPasses bounds how deep a chain of discovered artists is followed. +const maxCleanupPasses = 20 + // resolveArtists returns the artist id of each candidate, creating the ones that // are not known. A candidate that is the artist itself, or an artist already // seen earlier in the list, gets 0. @@ -281,6 +301,20 @@ func insertDiscovered(ctx context.Context, tx bun.Tx, name, mbid string) (int64, return id, nil } +// requireArtist returns discovery.ErrNotFound when the artist is gone, as it can be +// when it was removed while a provider was being asked about it. +func requireArtist(ctx context.Context, tx bun.Tx, id int64) error { + var one int + err := tx.QueryRowContext(ctx, "SELECT 1 FROM artists WHERE id = ?", id).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return discovery.ErrNotFound + } + if err != nil { + return fmt.Errorf("check artist: %w", err) + } + return nil +} + // artistWritesLock is the key of the PostgreSQL advisory lock that serializes the // creation of artists. SQLite needs none: its transactions already run one at a time. const artistWritesLock = 0x534f4e4441 // "SONDA" @@ -338,6 +372,13 @@ func (r *DiscoveryRepo) Tags(ctx context.Context, id int64, provider string) (di // SaveTags replaces the tags a provider gave to an artist and records the answer. func (r *DiscoveryRepo) SaveTags(ctx context.Context, id int64, provider string, at time.Time, found bool, tags []discovery.Tag) error { return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Artists can be removed by a purge or a cleanup: take the same turn they do. + if err := lockArtistWrites(ctx, tx); err != nil { + return err + } + if err := requireArtist(ctx, tx, id); err != nil { + return err + } if _, err := tx.ExecContext(ctx, "DELETE FROM artist_tags WHERE artist_id = ? AND provider = ?", id, provider); err != nil { return fmt.Errorf("clear tags: %w", err) } @@ -411,6 +452,11 @@ func (r *DiscoveryRepo) FileGenres(ctx context.Context, artistID int64) ([]disco func (r *DiscoveryRepo) Purge(ctx context.Context, provider string) (providers.Purged, error) { var out providers.Purged err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Artists are removed here: take the turn the scanner and discovery take, + // or a file could be matched to an artist this transaction then deletes. + if err := lockArtistWrites(ctx, tx); err != nil { + return err + } for _, step := range []struct { what string query string diff --git a/internal/storage/discovery_test.go b/internal/storage/discovery_test.go index fde9592..03c61b3 100644 --- a/internal/storage/discovery_test.go +++ b/internal/storage/discovery_test.go @@ -935,3 +935,96 @@ func TestReplacingAListRemovesTheArtistsNoOneNamesAnymore(t *testing.T) { } }) } + +// ---- review: purge, cleanup chains and deleted artists ---- + +// A purge deleting a discovered artist while a scan adopts it must not leave the +// scan with the id of an artist that is gone. +func TestPurgeAndAdoptionDoNotCollide(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + radiohead := artistID(t, db, "Radiohead") + + errs := make(chan error, 100) + for round := 0; round < 25; round++ { + name := fmt.Sprintf("Collide %d", round) + if err := repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{ + {Name: name, MBID: fmt.Sprintf("dddddddd-0000-4000-8000-%012d", round), Match: 1}}); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + if _, err := repo.Purge(ctx, "lastfm"); err != nil { + errs <- fmt.Errorf("Purge: %w", err) + } + }() + go func() { + defer wg.Done() + f := pf(fmt.Sprintf("col/%d.flac", round), fmt.Sprintf("Collide song %d", round), "", "", 0, 0, "flac", name) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + errs <- fmt.Errorf("SaveFile: %w", err) + } + }() + wg.Wait() + } + close(errs) + for err := range errs { + t.Error(err) + } + }) +} + +// An artist that was only a source of a list, and that no list names any more, +// goes too, and so do the artists only its own list kept alive. +func TestAChainOfDiscoveredArtistsIsCleanedUpTogether(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + radiohead := artistID(t, db, "Radiohead") + + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Dee", Match: 1}, {Name: "Keep", Match: 0.5}}) + dee := artistID(t, db, "Dee") + _ = repo.SaveSimilar(ctx, dee, "lastfm", at, true, []lastfm.Similar{{Name: "Eee", Match: 1}}) + eee := artistID(t, db, "Eee") + _ = repo.SaveSimilar(ctx, eee, "lastfm", at, true, []lastfm.Similar{{Name: "Fff", Match: 1}}) + + if err := repo.SaveSimilar(ctx, radiohead, "lastfm", at.Add(time.Hour), true, []lastfm.Similar{{Name: "Keep", Match: 1}}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"Dee", "Eee", "Fff"} { + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name = '"+name+"'"); n != 0 { + t.Errorf("%s is kept alive only by a list nobody can reach, and is still there", name) + } + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name = 'Keep'"); n != 1 { + t.Error("an artist the list still names was removed") + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_similar"); n != 1 { + t.Errorf("edges left = %d, want 1", n) + } + }) +} + +// Writing about an artist that was removed meanwhile is "not found", not an +// obscure foreign key error. +func TestSavingForAnArtistThatIsGoneIsNotFound(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + if err := repo.SaveSimilar(ctx, 99999, "lastfm", at, true, []lastfm.Similar{{Name: "X", Match: 1}}); !errors.Is(err, discovery.ErrNotFound) { + t.Errorf("SaveSimilar: %v", err) + } + if err := repo.SaveTags(ctx, 99999, "lastfm", at, true, []discovery.Tag{{Name: "x", Weight: 1}}); !errors.Is(err, discovery.ErrNotFound) { + t.Errorf("SaveTags: %v", err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name = 'X'"); n != 0 { + t.Error("an artist was created for a list nobody can read") + } + }) +} diff --git a/internal/storage/library.go b/internal/storage/library.go index 406472f..1af6dfb 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -505,18 +505,13 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6 return found.ID, nil } -// replaceGenres makes the genres of a track those the file says now. +// replaceGenres makes the genres of a track those the file says now. The reader +// already cleaned them; cleaning is idempotent, so a store used another way is safe too. func replaceGenres(ctx context.Context, tx bun.Tx, trackID int64, genres []string) error { if _, err := tx.ExecContext(ctx, "DELETE FROM track_genres WHERE track_id = ?", trackID); err != nil { return fmt.Errorf("clear genres: %w", err) } - seen := map[string]bool{} - for _, g := range genres { - g = strings.ToLower(strings.TrimSpace(g)) - if g == "" || seen[g] || len(seen) == library.MaxGenresPerTrack { - continue - } - seen[g] = true + for _, g := range library.CleanGenres(genres) { if _, err := tx.ExecContext(ctx, "INSERT INTO track_genres (track_id, genre) VALUES (?, ?)", trackID, g); err != nil { return fmt.Errorf("insert genre: %w", err) }