From 5fecd394e43b04d63b118a2fd3770631146660d1 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 09:21:11 +0100 Subject: [PATCH] fix(images): purge removes every archive cover, the id of an emptied album moves on - The covers that come from the Cover Art Archive now live in a folder of their own (images/album-remote). Taking the provider's data back deletes that folder, so nothing from the archive is left whatever became of its records: a rescan makes a record stale and a later search that finds nothing replaces it, and neither of those used to remove the files. A cover that moves between the user's files and the archive, or goes away, no longer leaves the old files behind. - A redirect to a place that is not allowed is an outage (502, not remembered, an older cover served), not "no cover" for a day for every album. A picture over the limit is still "no cover". - A track that moves to another album (a fixed title) leaves the first with no track: it gives up its release group id, so the album the track went to can take it. An album that has an id keeps it; an id another album owns is not given twice. - Waiting for a local slot is no longer reported as the provider being unreachable, and a build whose reads were purged three times is not served with files that were never written. - README and spec no longer say covers never contact anybody. - Tests: the new folder and the purge after a rescan, files left by a replaced cover, the local wait, repeated purges, a refused redirect, the update path with a taken id and the emptied album. --- README.md | 2 +- cmd/sondad/main.go | 8 +- internal/api/openapi/openapi.yaml | 2 +- internal/images/album_remote_test.go | 96 ++++++++++++++++++-- internal/images/artist.go | 42 +++++---- internal/images/service.go | 74 ++++++++++++--- internal/providers/coverart/coverart.go | 9 +- internal/providers/coverart/coverart_test.go | 8 +- internal/storage/images.go | 20 ---- internal/storage/images_test.go | 4 - internal/storage/library.go | 48 ++++++++-- internal/storage/library_test.go | 68 ++++++++++++++ 12 files changed, 297 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index c2de87b..873606b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers). - **Artist descriptions.** `GET /artists/{id}/description` shows what Wikipedia says about an artist: a short text, with the page it comes from, a link to it and the licence (CC BY-SA 4.0), which you must show next to it. With the `wikipedia` provider and the `musicbrainz` one switched on (the link to Wikidata comes from MusicBrainz), Sonda follows ids and never a name: the artist's MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in the first of your languages (`[wikipedia] languages`, the code of that Wikipedia, for example `["it", "en"]`) that has one. An artist without a MusicBrainz id has no description. It is kept for 30 days and removed with the provider's data. See [External providers](#external-providers). - **Artist photos.** `GET /artists/{id}/image` serves a free picture of an artist from Wikimedia Commons, and `GET /artists/{id}/image/credit` says whose it is and under which licence, which you must show next to it. Only free licences are kept (public domain, CC0, CC BY, CC BY-SA): non-commercial, no-derivatives, fair-use and unknown ones are refused, because Sonda stores a copy (`?size=small` is 300 pixels, `large` 1000) in the data folder. With the `commons` provider and the `musicbrainz` one switched on, Sonda follows ids and never a name: the artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the picture Wikidata names for it, and Commons for its licence. Many artists will have none. Kept for 30 days (a week when there is none) and removed, files included, with the provider's data. See [External providers](#external-providers). -- **Album covers.** `GET /albums/{id}/cover` serves the cover of an album from your own files: the picture embedded in them, or else a `cover`, `folder`, `front`, `album` or `albumart` image in the album's folder. It is built the first time it is asked for (a few at a time), scaled (`?size=small` is 300 pixels, `large` 1000) and kept in the data folder (`storage.data_dir`, you can delete it: covers are built again). A picture you replace in a folder is noticed within a week, or at once after a full rescan. When your files have no picture at all and an admin switched on the `coverartarchive` provider, the cover is asked from the Cover Art Archive by the MusicBrainz release group id that the tags of the album's files carry (`MUSICBRAINZ_RELEASEGROUPID`; run a full rescan to read it for music you already indexed); your own files always come first, and only that id leaves the server. For an album split in `CD1`/`Disc 2` folders, the picture in the folder above is used. Nothing is sent anywhere. It also takes `?access_token=`, so an `` element can use it. +- **Album covers.** `GET /albums/{id}/cover` serves the cover of an album from your own files: the picture embedded in them, or else a `cover`, `folder`, `front`, `album` or `albumart` image in the album's folder. It is built the first time it is asked for (a few at a time), scaled (`?size=small` is 300 pixels, `large` 1000) and kept in the data folder (`storage.data_dir`, you can delete it: covers are built again). A picture you replace in a folder is noticed within a week, or at once after a full rescan. When your files have no picture at all and an admin switched on the `coverartarchive` provider, the cover is asked from the Cover Art Archive by the MusicBrainz release group id that the tags of the album's files carry (`MUSICBRAINZ_RELEASEGROUPID`; run a full rescan to read it for music you already indexed); your own files always come first, and only that id leaves the server. For an album split in `CD1`/`Disc 2` folders, the picture in the folder above is used. Nothing is sent anywhere unless you switched on the `coverartarchive` provider (see [External providers](#external-providers)). It also takes `?access_token=`, so an `` element can use it. - **Wishlist and "not interested".** `PUT /wishlist/{artist}` puts an artist on your wishlist (`GET /wishlist` lists it, `DELETE` takes it off), and `POST /discover/{artist}/dismiss` hides an artist from your Discover for good (`DELETE` brings it back, `GET /discover/dismissed` lists them). An artist is wished or dismissed, never both, and both lists are yours alone. Discover marks the wished artists it shows (`wishlisted`) and never suggests a dismissed one. Nothing is downloaded or sent anywhere: the wishlist is a list for you, and for a download client you may set up later. An artist that only Last.fm named is kept while you have it marked (your lists still show it, but other users cannot see that it exists), and goes with the rest of Last.fm's data if an admin removes Last.fm's data. Removing MusicBrainz's data does not touch it. - **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, Discover may start from the artists you listen to most, and their names are sent to Last.fm (only if an admin switched it on). What Last.fm answers is kept in the shared database like any other lookup, so another local user can see, from `retrieved_at` of `GET /artists/{id}/similar`, that someone looked up an artist, but not who. - **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights, and the genres of the artist's own files, read offline. See [External providers](#external-providers). diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index cd3fddd..287162b 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -212,16 +212,12 @@ func run() error { return out, nil case providers.CoverArtArchive: pictures.BeginPurge() - // The records say which albums have a cover from the archive: read them before they go. - ids, err := imageRepo.OwnersBySource(ctx, "album", images.SourceCoverArt) - if err != nil { - return providers.Purged{}, err - } out, err := discoveryRepo.Purge(ctx, name) if err != nil { return out, err } - if _, err := pictures.RemoveAlbumFiles(ids); err != nil { + // The covers that came from the archive have a folder of their own: all of it goes. + if _, err := pictures.RemoveRemoteAlbumFiles(); err != nil { return out, fmt.Errorf("remove the covers: %w", err) } return out, nil diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 7914424..83c730d 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -1169,7 +1169,7 @@ paths: (`.jpg`, `.jpeg` or `.png`, any case) in the album's folder, in that order of names. It is built the first time it is asked for (that request can take a moment), scaled to fit and saved in the server's data folder, and served from there afterwards. It never - contacts anybody. Pictures that are too big (over 20 MB or 16 million pixels) or are not + contacts anybody (unless the `coverartarchive` provider is on, see below). Pictures that are too big (over 20 MB or 16 million pixels) or are not pictures are ignored, and only files inside the library are read. For an album split in folders named like `CD1` or `Disc 2`, the picture in the folder diff --git a/internal/images/album_remote_test.go b/internal/images/album_remote_test.go index 50e4310..4a626cf 100644 --- a/internal/images/album_remote_test.go +++ b/internal/images/album_remote_test.go @@ -79,7 +79,7 @@ func TestAnAlbumWithNoPictureInItsFilesGetsTheCoverOfTheArchiveAndKeepsIt(t *tes if n := atomic.LoadInt32(&r.remote.calls); n != 1 { t.Errorf("the archive was asked %d times, want 1: the cover is stored", n) } - info, err := os.Stat(filepath.Join(r.data, "images", "album", "1-large.jpg")) + info, err := os.Stat(filepath.Join(r.data, "images", "album-remote", "1-large.jpg")) if err != nil || info.Mode().Perm()&0o077 != 0 { t.Errorf("file: %v %v", info, err) } @@ -158,26 +158,104 @@ func TestAPurgeDuringTheReadOfACoverIsNotUndoneByIt(t *testing.T) { } } -func TestPurgingTheArchiveCoversRemovesThoseFilesOnly(t *testing.T) { +func TestPurgingTheArchiveCoversRemovesAllOfThemWhateverBecameOfTheirRecords(t *testing.T) { r := newRemoteRig(t) r.rig.read(t, 1, Large) // from the archive r.src.files[2] = []File{{Root: r.root, RelPath: "Album One/01.mp3"}} r.write(t, "Album One/folder.jpg", jpegBytes(t, 300, 300, color.White)) r.remote.set(Found{}, ErrNoImage) r.rig.read(t, 2, Large) // from the folder - n, err := r.svc.RemoveAlbumFiles([]int64{1, 99}) + + // The records of the archive cover change (a rescan makes it stale) and the files must still go. + r.store.clear("album", 1) + n, err := r.svc.RemoveRemoteAlbumFiles() if err != nil || n != 2 { - t.Fatalf("removed %d files err = %v, want the two sizes of album 1", n, err) + t.Fatalf("removed %d files err = %v, want the two sizes of the archive cover", n, err) } - if _, err := os.Stat(filepath.Join(r.data, "images", "album", "1-large.jpg")); !os.IsNotExist(err) { - t.Error("the archive cover is still there") + if _, err := os.Stat(filepath.Join(r.data, "images", "album-remote")); !os.IsNotExist(err) { + t.Error("the folder of the archive covers is still there") } if _, err := os.Stat(filepath.Join(r.data, "images", "album", "2-large.jpg")); err != nil { t.Errorf("the cover from the folder went too: %v", err) } - // Ids that are not numbers of an album can never reach outside the folder. - if n, err := r.svc.RemoveAlbumFiles([]int64{-1, 0}); err != nil || n != 0 { - t.Errorf("bad ids: %d %v", n, err) + if n, err := r.svc.RemoveRemoteAlbumFiles(); err != nil || n != 0 { + t.Errorf("again: %d %v", n, err) + } +} + +func TestACoverFromTheArchiveLivesApartFromTheOnesFromTheUsersFiles(t *testing.T) { + r := newRemoteRig(t) + r.rig.read(t, 1, Large) + if _, err := os.Stat(filepath.Join(r.data, "images", "album-remote", "1-large.jpg")); err != nil { + t.Errorf("the archive cover is not in its own folder: %v", err) + } + if _, err := os.Stat(filepath.Join(r.data, "images", "album", "1-large.jpg")); !os.IsNotExist(err) { + t.Error("the archive cover is among the covers from the user's files") + } +} + +func TestACoverFromTheFilesReplacingOneFromTheArchiveLeavesNoOrphan(t *testing.T) { + r := newRemoteRig(t) + r.rig.read(t, 1, Large) + r.write(t, "Album One/cover.jpg", jpegBytes(t, 300, 300, color.RGBA{0, 0, 200, 255})) + r.store.clear("album", 1) // a file of the album changed + r.rig.read(t, 1, Large) + if _, err := os.Stat(filepath.Join(r.data, "images", "album-remote", "1-large.jpg")); !os.IsNotExist(err) { + t.Error("the archive cover stayed on disk after the files got their own") + } + // And the other way round: no picture any more (the archive says so) removes the old file too. + r2 := newRemoteRig(t) + r2.rig.read(t, 1, Large) + r2.now = r2.now.Add(CoverTTL + time.Hour) + r2.remote.set(Found{}, ErrNoImage) + if _, err := r2.svc.AlbumCover(context.Background(), 1, Large); !errors.Is(err, ErrNoCover) { + t.Fatalf("err = %v", err) + } + if entries, _ := os.ReadDir(filepath.Join(r2.data, "images", "album-remote")); len(entries) != 0 { + t.Errorf("%d files left for a cover that is gone", len(entries)) + } +} + +func TestAWaitForALocalSlotIsNotAnOutageOfTheProvider(t *testing.T) { + r := newRemoteRig(t) + r.svc.buildTimeout = 40 * time.Millisecond + for i := 0; i < MaxParallelBuilds; i++ { + r.svc.slots <- struct{}{} // every local slot is taken + } + _, err := r.svc.AlbumCover(context.Background(), 1, Large) + if errors.Is(err, ErrSourceUnavailable) { + t.Errorf("err = %v: waiting for a local slot says nothing about the provider", err) + } + if err == nil { + t.Error("no error although the build could not start") + } + if n := atomic.LoadInt32(&r.remote.calls); n != 0 { + t.Errorf("the provider was asked %d times", n) + } +} + +func TestPurgesThatKeepHappeningDoNotLeaveARecordWithoutFiles(t *testing.T) { + r := newRemoteRig(t) + r.remote.gate = make(chan struct{}) + done := make(chan error, 1) + go func() { _, err := r.svc.AlbumCover(context.Background(), 1, Large); done <- err }() + for i := 0; i < 3; i++ { + time.Sleep(30 * time.Millisecond) + r.svc.BeginPurge() + if i == 2 { + close(r.remote.gate) + } + } + err := <-done + if err == nil { + if _, serr := os.Stat(filepath.Join(r.data, "images", "album-remote", "1-large.jpg")); serr != nil { + t.Errorf("a cover is served but its file was never written: %v", serr) + } + } + if rec := r.store.recs["album/1"]; rec.Status == StatusOK { + if _, serr := os.Stat(filepath.Join(r.data, "images", "album-remote", "1-large.jpg")); serr != nil { + t.Errorf("an ok record without its file: %v", serr) + } } } diff --git a/internal/images/artist.go b/internal/images/artist.go index 0712df0..0a5aa38 100644 --- a/internal/images/artist.go +++ b/internal/images/artist.go @@ -98,6 +98,7 @@ func (s *Service) buildArtist(artistID int64) (Record, error) { ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout) defer cancel() var rec Record + purged := false for attempt := 0; attempt < 3; attempt++ { cur, _, err := s.store.Get(ctx, kindArtist, artistID) if err != nil { @@ -120,6 +121,7 @@ func (s *Service) buildArtist(artistID int64) (Record, error) { rec.Gen = cur.Gen saved, err := s.commit(ctx, kindArtist, artistID, rec, pic, epoch) if errors.Is(err, errPurged) { + purged = true continue } if err != nil { @@ -129,6 +131,9 @@ func (s *Service) buildArtist(artistID int64) (Record, error) { return rec, nil } } + if purged { + return Record{}, ErrSourceUnavailable // its files were never written: there is nothing to serve + } return rec, nil } @@ -188,26 +193,31 @@ func (s *Service) RemoveArtistFiles() (int, error) { return n, nil } -// RemoveAlbumFiles deletes the stored covers of the given albums (both sizes), and says -// how many files went. It is what taking the cover provider's data back needs, for the -// albums whose cover came from it; ids that are not an album's are ignored. -func (s *Service) RemoveAlbumFiles(albumIDs []int64) (int, error) { - dir := filepath.Join(s.dir, "images", kindAlbum) +// RemoveRemoteAlbumFiles deletes the stored covers that came from the archive, and only +// those, and says how many files went. They have a folder of their own, so this does not +// depend on what became of their records (a rescan makes a record stale, a later search that +// finds nothing replaces it): nothing from the archive is left. It is what taking the cover +// provider's data back needs, together with forgetting the records. +func (s *Service) RemoveRemoteAlbumFiles() (int, error) { + dir := filepath.Join(s.dir, "images", folderAlbumRemote) + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("list the archive covers: %w", err) + } n := 0 - for _, id := range albumIDs { - if id <= 0 { + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".jpg") { continue } - for _, size := range []Size{Large, Small} { - err := os.Remove(s.filePath(kindAlbum, id, size)) - switch { - case err == nil: - n++ - case errors.Is(err, os.ErrNotExist): - default: - return n, fmt.Errorf("remove a cover in %s: %w", dir, err) - } + if err := os.Remove(filepath.Join(dir, e.Name())); err != nil { + return n, fmt.Errorf("remove a cover: %w", err) } + n++ } + _ = os.Remove(dir) // goes too when nothing else is in it + s.tidied.Delete(dir) return n, nil } diff --git a/internal/images/service.go b/internal/images/service.go index 52a30a1..d00fe5a 100644 --- a/internal/images/service.go +++ b/internal/images/service.go @@ -26,6 +26,8 @@ import ( const ( kindAlbum = "album" kindArtist = "artist" + + folderAlbumRemote = "album-remote" // the covers from the archive ) // What is stored about a picture, and where it came from. @@ -260,12 +262,22 @@ func (s *Service) AlbumCover(ctx context.Context, albumID int64, size Size) (*Co return s.open(kindAlbum, albumID, size, built) } -func (s *Service) filePath(kind string, id int64, size Size) string { - return filepath.Join(s.dir, "images", kind, strconv.FormatInt(id, 10)+"-"+string(size)+".jpg") +// folderOf is the folder (under images/) where the files of a picture are kept. The covers +// that came from the archive have a folder of their own, so that taking the provider's data +// back removes exactly them whatever became of their records. +func folderOf(kind, source string) string { + if kind == kindAlbum && source == SourceCoverArt { + return folderAlbumRemote + } + return kind +} + +func (s *Service) filePath(folder string, id int64, size Size) string { + return filepath.Join(s.dir, "images", folder, strconv.FormatInt(id, 10)+"-"+string(size)+".jpg") } func (s *Service) open(kind string, id int64, size Size, rec Record) (*Cover, error) { - f, err := os.Open(s.filePath(kind, id, size)) + f, err := os.Open(s.filePath(folderOf(kind, rec.Source), id, size)) if err != nil { return nil, err } @@ -288,6 +300,7 @@ func (s *Service) build(albumID int64) (Record, error) { ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout) defer cancel() var rec Record + purged := false for attempt := 0; attempt < 3; attempt++ { cur, _, err := s.store.Get(ctx, kindAlbum, albumID) if err != nil { @@ -301,16 +314,12 @@ func (s *Service) build(albumID int64) (Record, error) { var pic *Processed rec, pic, err = s.search(ctx, albumID, files) if err != nil { - // Nobody calls this context: when it ends it is the build that ran out of time, - // which is an outage of the provider, not an internal error. - if ctx.Err() != nil && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) { - err = fmt.Errorf("%w: %w", ErrSourceUnavailable, err) - } return Record{}, err } rec.Gen = cur.Gen saved, err := s.commit(ctx, kindAlbum, albumID, rec, pic, epoch) if errors.Is(err, errPurged) { + purged = true continue } if err != nil { @@ -320,6 +329,9 @@ func (s *Service) build(albumID int64) (Record, error) { return rec, nil } } + if purged { + return Record{}, ErrSourceUnavailable // its files were never written: there is nothing to serve + } return rec, nil // it kept changing; this one is served and the record stays stale } @@ -362,18 +374,26 @@ func (s *Service) searchFiles(ctx context.Context, files []File) (Processed, str // searchRemote asks the provider of covers. "There is none" (no picture, no id, the // provider is off, a picture that is not one) is an answer; a failure is returned. func (s *Service) searchRemote(ctx context.Context, albumID int64) (Processed, Credit, bool, error) { + // This context is nobody's: when it ends here, the build ran out of time waiting for the + // provider, which is an outage of the provider and not an internal error. + outage := func(err error) error { + if ctx.Err() != nil && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) { + return fmt.Errorf("%w: %w", ErrSourceUnavailable, err) + } + return err + } select { case s.artistSl <- struct{}{}: // the slots of the work that waits for the network defer func() { <-s.artistSl }() case <-ctx.Done(): - return Processed{}, Credit{}, false, ctx.Err() + return Processed{}, Credit{}, false, outage(ctx.Err()) } found, err := s.remote.AlbumImage(ctx, albumID) switch { case errors.Is(err, ErrNoImage), errors.Is(err, ErrNoID), errors.Is(err, ErrProviderDisabled): return Processed{}, Credit{}, false, nil case err != nil: - return Processed{}, Credit{}, false, err + return Processed{}, Credit{}, false, outage(err) } p, err := ProcessLimited(found.Data, MaxArtistPixels) if err != nil { @@ -405,8 +425,9 @@ func (s *Service) commit(ctx context.Context, kind string, id int64, rec Record, if s.purges != epoch { return false, errPurged } + folder := folderOf(kind, rec.Source) if pic != nil { - if err := s.write(kind, id, *pic); err != nil { + if err := s.write(folder, id, *pic); err != nil { return false, err } } @@ -414,9 +435,34 @@ func (s *Service) commit(ctx context.Context, kind string, id int64, rec Record, if err != nil { return false, fmt.Errorf("save picture record: %w", err) } + if saved { + // What the record no longer describes must not stay on disk: the files of the other + // folder when the cover moved between the user's files and the archive, and all of + // them when there is no cover any more. + for _, f := range foldersOf(kind) { + if pic == nil || f != folder { + s.removeFiles(f, id) + } + } + } return saved, nil } +// foldersOf lists the folders a kind of picture can be kept in. +func foldersOf(kind string) []string { + if kind == kindAlbum { + return []string{kindAlbum, folderAlbumRemote} + } + return []string{kind} +} + +// removeFiles deletes both sizes of a picture, if they are there. +func (s *Service) removeFiles(folder string, id int64) { + for _, size := range []Size{Large, Small} { + _ = os.Remove(s.filePath(folder, id, size)) + } +} + // find returns the first usable picture: embedded in one of the first files, then an // image in the folder of one of them. func (s *Service) find(files []File) (Processed, string, bool) { @@ -559,8 +605,8 @@ func readFolderPicture(rootDir, dir, name string) ([]byte, bool) { // write stores both sizes. Each goes to a temporary file first and is renamed, so a // reader never sees half a picture. -func (s *Service) write(kind string, id int64, p Processed) error { - dir := filepath.Dir(s.filePath(kind, id, Large)) +func (s *Service) write(folder string, id int64, p Processed) error { + dir := filepath.Dir(s.filePath(folder, id, Large)) if err := os.MkdirAll(dir, 0o700); err != nil { return fmt.Errorf("create the data folder: %w", err) } @@ -579,7 +625,7 @@ func (s *Service) write(kind string, id int64, p Processed) error { _ = os.Remove(tmp.Name()) return fmt.Errorf("write a picture file: %w", errors.Join(werr, serr, cerr)) } - if err := os.Rename(tmp.Name(), s.filePath(kind, id, size)); err != nil { + if err := os.Rename(tmp.Name(), s.filePath(folder, id, size)); err != nil { _ = os.Remove(tmp.Name()) return fmt.Errorf("store a picture file: %w", err) } diff --git a/internal/providers/coverart/coverart.go b/internal/providers/coverart/coverart.go index 9341ebd..849a6e1 100644 --- a/internal/providers/coverart/coverart.go +++ b/internal/providers/coverart/coverart.go @@ -105,10 +105,11 @@ func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { switch { case errors.As(err, &limited): return nil, ErrRateLimited - // A redirect that is not allowed, or a picture over the limit, is a fault of that - // answer and not an outage: there is no usable cover. - case errors.Is(err, outbound.ErrRedirect), errors.Is(err, outbound.ErrResponseTooLarge): - return nil, fmt.Errorf("%w: the answer cannot be followed or is too big", ErrUnusable) + // A picture over the limit is a fault of that cover: there is no usable cover. A + // redirect somewhere we do not follow is another matter: it says the archive changed, + // which is a problem of the service for now, not "no cover" for every album for a day. + case errors.Is(err, outbound.ErrResponseTooLarge): + return nil, fmt.Errorf("%w: the answer is too big", ErrUnusable) } return nil, fmt.Errorf("%w: %w", ErrUnavailable, err) } diff --git a/internal/providers/coverart/coverart_test.go b/internal/providers/coverart/coverart_test.go index 1e85ba8..e2c73d1 100644 --- a/internal/providers/coverart/coverart_test.go +++ b/internal/providers/coverart/coverart_test.go @@ -140,13 +140,15 @@ func TestFrontErrors(t *testing.T) { } } -func TestARedirectToAPlaceThatIsNotAllowedOrAHugePictureIsUnusable(t *testing.T) { +func TestARedirectToAPlaceThatIsNotAllowedIsAnOutageAndAHugePictureIsUnusable(t *testing.T) { redirect := newServer(t, func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "https://evil.example/", http.StatusFound) }) c := New(outbound.New(outbound.Options{UserAgent: "x", Retries: outbound.NoRetries, FollowRedirect: RedirectAllowed}), redirect.URL+"/") - if _, err := c.Front(context.Background(), rg); !errors.Is(err, ErrUnusable) { - t.Errorf("redirect: err = %v, want ErrUnusable", err) + // A redirect somewhere we do not follow says the archive changed: an outage for now, not "no + // cover", which would be remembered for a day for every album. + if _, err := c.Front(context.Background(), rg); !errors.Is(err, ErrUnavailable) || errors.Is(err, ErrUnusable) { + t.Errorf("redirect: err = %v, want ErrUnavailable", err) } huge := newServer(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(make([]byte, 5000)) }) c = New(outbound.New(outbound.Options{UserAgent: "x", Retries: outbound.NoRetries, MaxBody: 1000}), huge.URL+"/") diff --git a/internal/storage/images.go b/internal/storage/images.go index a405b22..998f548 100644 --- a/internal/storage/images.go +++ b/internal/storage/images.go @@ -127,23 +127,3 @@ func (r *ImageRepo) AlbumMBID(ctx context.Context, albumID int64) (string, error } return m.String, nil } - -// OwnersBySource returns the ids of the owners (albums, artists) whose stored picture came -// from the given source, in order. A purge needs them to remove the files too. -func (r *ImageRepo) OwnersBySource(ctx context.Context, kind, source string) ([]int64, error) { - rows, err := r.db.QueryContext(ctx, - "SELECT owner_id FROM images WHERE kind = ? AND source = ? AND status = 'ok' ORDER BY owner_id", kind, source) - if err != nil { - return nil, fmt.Errorf("list pictures: %w", err) - } - defer rows.Close() - var out []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err - } - out = append(out, id) - } - return out, rows.Err() -} diff --git a/internal/storage/images_test.go b/internal/storage/images_test.go index 84f3374..10c65e9 100644 --- a/internal/storage/images_test.go +++ b/internal/storage/images_test.go @@ -354,10 +354,6 @@ func TestPurgingTheArchiveForgetsOnlyTheCoversThatCameFromIt(t *testing.T) { if _, err := repo.Save(ctx, "album", 5, images.Record{Status: images.StatusNone, CheckedAt: at}); err != nil { t.Fatal(err) } - ids, err := repo.OwnersBySource(ctx, "album", images.SourceCoverArt) - if err != nil || len(ids) != 2 || ids[0] != 1 || ids[1] != 2 { - t.Fatalf("owners = %v err = %v, want 1 and 2", ids, err) - } out, err := NewDiscoveryRepo(db).Purge(ctx, "coverartarchive") if err != nil || out.Images != 2 { t.Fatalf("purge = %+v err = %v, want 2", out, err) diff --git a/internal/storage/library.go b/internal/storage/library.go index bda9ad0..0ce6c1e 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -184,6 +184,16 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P WherePK().Exec(ctx); err != nil { return fmt.Errorf("update track: %w", err) } + // A track that moved to another album may have left the first with none: that one gives + // up the id it holds, which the album the track went to can then take. + if oldAlbum.Valid && oldAlbum.Int64 != albumID { + if err := releaseEmptyAlbumID(ctx, tx, oldAlbum.Int64); err != nil { + return err + } + if err := claimAlbumID(ctx, tx, albumID, f.AlbumMBID); err != nil { + return err + } + } file.Format, file.SizeBytes, file.MTimeUnix = f.Format, f.Size, f.MTimeUnix file.SampleRate, file.BitRate = f.SampleRate, f.BitRate file.MissingAt = sql.NullInt64{} @@ -531,6 +541,36 @@ func replaceGenres(ctx context.Context, tx bun.Tx, trackID int64, genres []strin return nil } +// claimAlbumID gives an album the release group id a file carries, when it has none and no +// other album owns the id (it is unique). It never replaces an id the album has, and a +// refusal is not an error. +func claimAlbumID(ctx context.Context, tx bun.Tx, albumID int64, mbid string) error { + if mbid == "" || albumID == 0 { + return nil + } + if _, err := tx.ExecContext(ctx, + "UPDATE albums SET mbid = ? WHERE id = ? AND mbid IS NULL AND NOT EXISTS (SELECT 1 FROM albums o WHERE o.mbid = ?)", + mbid, albumID, mbid); err != nil { + return fmt.Errorf("set album id: %w", err) + } + return nil +} + +// releaseEmptyAlbumID takes the release group id off an album that has no track any more. +// Albums are never deleted, so otherwise the id would stay on that empty album for good and +// the album the tracks moved to (a title that was fixed) could never have it. +func releaseEmptyAlbumID(ctx context.Context, tx bun.Tx, albumID int64) error { + if albumID == 0 { + return nil + } + if _, err := tx.ExecContext(ctx, + "UPDATE albums SET mbid = NULL WHERE id = ? AND mbid IS NOT NULL AND NOT EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = ?)", + albumID, albumID); err != nil { + return fmt.Errorf("release album id: %w", err) + } + return nil +} + func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64, mbid string) (int64, error) { var a albumModel err := tx.NewSelect().Model(&a). @@ -539,12 +579,8 @@ func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID in if err == nil { // An album that has no id takes the one a file brings (a rescan after tagging). One that // has an id keeps it, and an id that another album already has (it is unique) is not given twice. - if mbid != "" && a.MBID == "" { - if _, err := tx.ExecContext(ctx, - "UPDATE albums SET mbid = ? WHERE id = ? AND mbid IS NULL AND NOT EXISTS (SELECT 1 FROM albums o WHERE o.mbid = ?)", - mbid, a.ID, mbid); err != nil { - return 0, fmt.Errorf("set album id: %w", err) - } + if err := claimAlbumID(ctx, tx, a.ID, mbid); err != nil { + return 0, err } return a.ID, nil } diff --git a/internal/storage/library_test.go b/internal/storage/library_test.go index 7c55a0e..9307482 100644 --- a/internal/storage/library_test.go +++ b/internal/storage/library_test.go @@ -553,3 +553,71 @@ func TestAnAlbumTakesTheReleaseGroupIDFromItsFiles(t *testing.T) { } }) } + +func TestAnAlbumWithoutAnIDCannotTakeOneThatAnotherAlbumHasAndNothingBreaks(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l, _ := lib.CreateLibrary(ctx, "m", "/lib/m") + const id = "11111111-1111-4111-8111-111111111111" + save := func(f library.ParsedFile) { + t.Helper() + if _, err := lib.SaveFile(ctx, l.ID, f, 0); err != nil { + t.Fatalf("SaveFile: %v", err) + } + } + save(withAlbumID(pf("a/1.flac", "One", "Owner", "Band", 1, 1, "flac", "Band"), id)) + save(pf("b/1.flac", "Two", "Waiting", "Band", 1, 1, "flac", "Band")) + // An album that already exists, has no id, and now meets one that is taken: the + // UPDATE must not violate the unique constraint, and must leave it without. + save(withAlbumID(pf("b/2.flac", "Three", "Waiting", "Band", 2, 1, "flac", "Band"), id)) + if got := albumMBID(t, db, "Waiting"); got != "" { + t.Errorf("Waiting = %q, want none", got) + } + if got := albumMBID(t, db, "Owner"); got != id { + t.Errorf("Owner = %q, want it to keep %q", got, id) + } + }) +} + +// Retagging a title leaves the old album with no track. Albums are never deleted, so its id +// must be given up, or the retagged album could never have it. +func TestTheIDOfAnAlbumWithNoTracksLeftPassesToTheRetaggedOne(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l, _ := lib.CreateLibrary(ctx, "m", "/lib/m") + const id = "11111111-1111-4111-8111-111111111111" + if _, err := lib.SaveFile(ctx, l.ID, withAlbumID(pf("a/1.flac", "One", "Typo Albm", "Band", 1, 1, "flac", "Band"), id), 0); err != nil { + t.Fatal(err) + } + var fileID int64 + if err := db.QueryRow("SELECT id FROM audio_files WHERE rel_path = 'a/1.flac'").Scan(&fileID); err != nil { + t.Fatal(err) + } + // The title is fixed: the same file now says another album, and carries the same id. + if _, err := lib.SaveFile(ctx, l.ID, withAlbumID(pf("a/1.flac", "One", "Fixed Album", "Band", 1, 1, "flac", "Band"), id), fileID); err != nil { + t.Fatalf("SaveFile: %v", err) + } + if got := albumMBID(t, db, "Fixed Album"); got != id { + t.Errorf("Fixed Album = %q, want %q", got, id) + } + if got := albumMBID(t, db, "Typo Albm"); got != "" { + t.Errorf("Typo Albm = %q: an album with no tracks must not hold the id", got) + } + // An album that still has tracks keeps its id. + if _, err := lib.SaveFile(ctx, l.ID, withAlbumID(pf("c/1.flac", "Two", "Keeps", "Band", 1, 1, "flac", "Band"), "22222222-2222-4222-8222-222222222222"), 0); err != nil { + t.Fatal(err) + } + var other int64 + if err := db.QueryRow("SELECT id FROM audio_files WHERE rel_path = 'c/1.flac'").Scan(&other); err != nil { + t.Fatal(err) + } + if _, err := lib.SaveFile(ctx, l.ID, pf("c/2.flac", "Three", "Keeps", "Band", 2, 1, "flac", "Band"), 0); err != nil { + t.Fatal(err) + } + if got := albumMBID(t, db, "Keeps"); got != "22222222-2222-4222-8222-222222222222" { + t.Errorf("Keeps = %q", got) + } + }) +}