- A panic while counting no longer leaves the job marked as running; a panic in the work of one item is a failure of that item, and a panic in the list ends the job with a reason. - A stop while the items are counted is recorded as a stop instead of a 500, and a count that fails leaves the status as the previous job left it. - An artist that lost its MusicBrainz id since it was listed is skipped by the pictures job, as the spec says, instead of counting as a failure. - A picture that cannot be used now (refused download, bytes that are not an image) is a failure when a good picture is stored, so a refresh never throws it away. - The outcomes the artist jobs report share their constants in the jobs package. - The spec no longer calls each job "the one request" that goes through the library. - The test that waited with a sleep for the count now waits for a signal.
274 lines
9.2 KiB
Go
274 lines
9.2 KiB
Go
package images
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// ArtistImage returns the picture of an artist in the given size. It is built from
|
|
// the free picture the source finds, the first time it is asked for, and kept in the
|
|
// data folder: 30 days, or a week when there is none. When the provider cannot be asked
|
|
// and an older picture is stored, the old one is served. Anything that is not an answer
|
|
// about the picture (the provider is off, the artist has no id, the network failed) is
|
|
// returned and not remembered, so the next request asks again.
|
|
func (s *Service) ArtistImage(ctx context.Context, artistID int64, size Size) (*Cover, error) {
|
|
if size != Small && size != Large {
|
|
return nil, fmt.Errorf("%w: %q", ErrInvalidSize, string(size))
|
|
}
|
|
if artistID <= 0 {
|
|
return nil, ErrArtistNotFound
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
rec, ok, err := s.store.Get(ctx, kindArtist, artistID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read picture record: %w", err)
|
|
}
|
|
if ok && rec.Status == StatusNone && s.now().Sub(rec.CheckedAt) < NoImageTTL {
|
|
return nil, ErrNoImage
|
|
}
|
|
if ok && rec.Status == StatusOK && s.now().Sub(rec.CheckedAt) < ArtistTTL {
|
|
if c, err := s.open(kindArtist, artistID, size, rec); err == nil {
|
|
return c, nil
|
|
}
|
|
}
|
|
|
|
built, err := s.buildArtistOnce(ctx, artistID)
|
|
if err != nil {
|
|
// An older picture is better than none when the provider cannot be asked now.
|
|
if (errors.Is(err, ErrProviderDisabled) || errors.Is(err, ErrSourceUnavailable)) && ok && rec.Status == StatusOK {
|
|
if c, err := s.open(kindArtist, artistID, size, rec); err == nil {
|
|
return c, nil
|
|
}
|
|
}
|
|
return nil, err
|
|
}
|
|
if built.Status == StatusNone {
|
|
return nil, ErrNoImage
|
|
}
|
|
return s.open(kindArtist, artistID, size, built)
|
|
}
|
|
|
|
// buildArtistOnce builds the picture of an artist: one build per artist, however many ask at
|
|
// once, on its own context. The caller may leave without stopping it.
|
|
func (s *Service) buildArtistOnce(ctx context.Context, artistID int64) (Record, error) {
|
|
done := s.flight.DoChan("artist/"+strconv.FormatInt(artistID, 10), func() (v any, err error) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
err = fmt.Errorf("building a picture panicked: %v", r)
|
|
}
|
|
}()
|
|
return s.buildArtist(artistID)
|
|
})
|
|
select {
|
|
case <-ctx.Done():
|
|
return Record{}, ctx.Err()
|
|
case r := <-done:
|
|
if r.Err != nil {
|
|
return Record{}, r.Err
|
|
}
|
|
return r.Val.(Record), nil
|
|
}
|
|
}
|
|
|
|
// EnsureArtistImage makes sure the picture of an artist has been looked for, and says
|
|
// whether there is one. A picture or an answer of "none" that is still good is left alone,
|
|
// unless refresh asks to search again. Whatever is not an answer about the picture (the
|
|
// provider is off, the artist is gone, the network failed) is returned and changes nothing:
|
|
// a refresh never replaces a stored picture by a failure.
|
|
func (s *Service) EnsureArtistImage(ctx context.Context, artistID int64, refresh bool) (bool, error) {
|
|
if artistID <= 0 {
|
|
return false, ErrArtistNotFound
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return false, err
|
|
}
|
|
if !refresh {
|
|
rec, ok, err := s.store.Get(ctx, kindArtist, artistID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("read picture record: %w", err)
|
|
}
|
|
if ok && rec.Status == StatusNone && s.now().Sub(rec.CheckedAt) < NoImageTTL {
|
|
return false, nil
|
|
}
|
|
if ok && rec.Status == StatusOK && s.now().Sub(rec.CheckedAt) < ArtistTTL {
|
|
if c, err := s.open(kindArtist, artistID, Large, rec); err == nil {
|
|
_ = c.File.Close()
|
|
return true, nil
|
|
}
|
|
}
|
|
}
|
|
built, err := s.buildArtistOnce(ctx, artistID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return built.Status == StatusOK, nil
|
|
}
|
|
|
|
// MaxArtistPixels is the largest copy of an artist's picture that is decoded. The copy
|
|
// asked from Commons is 960 pixels wide, so this leaves room for a very tall one.
|
|
const MaxArtistPixels = 4_000_000
|
|
|
|
// BeginPurge must be called before the pictures of a provider are taken back (their
|
|
// records forgotten and their files removed). A read that started before it is not kept:
|
|
// it starts again, and is refused if the provider is by then off.
|
|
func (s *Service) BeginPurge() {
|
|
s.purgeMu.Lock()
|
|
s.purges++
|
|
s.purgeMu.Unlock()
|
|
}
|
|
|
|
func (s *Service) purgeCount() uint64 {
|
|
s.purgeMu.RLock()
|
|
defer s.purgeMu.RUnlock()
|
|
return s.purges
|
|
}
|
|
|
|
// buildArtist asks the source, stores the picture and records the outcome. If the
|
|
// record was cleared or the pictures were taken back while it worked, what it found may
|
|
// be out of date: it starts again.
|
|
func (s *Service) buildArtist(artistID int64) (Record, error) {
|
|
if s.artists == nil {
|
|
return Record{}, ErrProviderDisabled
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout)
|
|
defer cancel()
|
|
var rec Record
|
|
purged := false
|
|
for attempt := 0; attempt < 3; attempt++ {
|
|
cur, had, err := s.store.Get(ctx, kindArtist, artistID)
|
|
if err != nil {
|
|
return Record{}, fmt.Errorf("read picture record: %w", err)
|
|
}
|
|
epoch := s.purgeCount()
|
|
var pic *Processed
|
|
rec, pic, err = s.searchArtist(ctx, artistID, had && cur.Status == StatusOK)
|
|
if errors.Is(err, errPurged) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
// Nobody calls this context: when it ends it is the build that ran out of time,
|
|
// which is an outage of the provider and not something to hide as an internal error.
|
|
if ctx.Err() != nil && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) {
|
|
err = fmt.Errorf("%w: %w", ErrSourceUnavailable, err)
|
|
}
|
|
return Record{}, err
|
|
}
|
|
rec.Gen = cur.Gen
|
|
saved, err := s.commit(ctx, kindArtist, artistID, rec, pic, epoch, false)
|
|
if errors.Is(err, errPurged) {
|
|
purged = true
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return Record{}, err
|
|
}
|
|
if saved {
|
|
return rec, nil
|
|
}
|
|
}
|
|
if purged {
|
|
return Record{}, ErrSourceUnavailable // its files were never written: there is nothing to serve
|
|
}
|
|
return rec, nil
|
|
}
|
|
|
|
// errPurged says the pictures were taken back since the build began.
|
|
var errPurged = errors.New("the pictures were taken back while this was being built")
|
|
|
|
// searchArtist asks the source and, for a picture, prepares it (nil when there is none).
|
|
// The files are written by saveArtist, together with the record, so that a purge cannot
|
|
// fall between them.
|
|
//
|
|
// keepGood says a good picture is stored: a download that cannot be used is then a failure,
|
|
// not "no picture", so that the picture is not thrown away for a bad answer.
|
|
func (s *Service) searchArtist(ctx context.Context, artistID int64, keepGood bool) (Record, *Processed, error) {
|
|
select {
|
|
case s.artistSl <- struct{}{}:
|
|
defer func() { <-s.artistSl }()
|
|
case <-ctx.Done():
|
|
return Record{}, nil, ctx.Err()
|
|
}
|
|
none := Record{Status: StatusNone, CheckedAt: s.now()}
|
|
found, err := s.artists.ArtistImage(ctx, artistID)
|
|
bad := fmt.Errorf("%w: what the provider sent cannot be used", ErrSourceUnavailable)
|
|
switch {
|
|
case errors.Is(err, ErrNoImage) && errors.Is(err, ErrBadPicture) && keepGood:
|
|
return Record{}, nil, bad
|
|
case errors.Is(err, ErrNoImage):
|
|
return none, nil, nil
|
|
case err != nil:
|
|
return Record{}, nil, err
|
|
}
|
|
p, err := ProcessLimited(found.Data, MaxArtistPixels)
|
|
if err != nil {
|
|
if keepGood {
|
|
return Record{}, nil, bad
|
|
}
|
|
return none, nil, nil // what the provider sent is not a usable picture: there is none to show
|
|
}
|
|
return s.record(p, SourceCommons, found.Credit, none.CheckedAt), &p, nil
|
|
}
|
|
|
|
// RemoveArtistFiles deletes the stored pictures of artists, and only those, and says how
|
|
// many files went. It is what taking the picture provider's data back needs, together with
|
|
// forgetting their records.
|
|
func (s *Service) RemoveArtistFiles() (int, error) {
|
|
dir := filepath.Join(s.dir, "images", kindArtist)
|
|
entries, err := os.ReadDir(dir)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return 0, nil
|
|
}
|
|
if err != nil {
|
|
return 0, fmt.Errorf("list the artist pictures: %w", err)
|
|
}
|
|
n := 0
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".jpg") {
|
|
continue
|
|
}
|
|
if err := os.Remove(filepath.Join(dir, e.Name())); err != nil {
|
|
return n, fmt.Errorf("remove a picture: %w", err)
|
|
}
|
|
n++
|
|
}
|
|
// The folder goes too when nothing else is in it (a temporary file may be).
|
|
_ = os.Remove(dir)
|
|
s.tidied.Delete(dir)
|
|
return n, nil
|
|
}
|
|
|
|
// RemoveRemoteAlbumFiles deletes the stored covers that came from the archive, and only
|
|
// those, and says how many files went. They have a folder of their own, so this does not
|
|
// depend on what became of their records (a rescan makes a record stale, a later search that
|
|
// finds nothing replaces it): nothing from the archive is left. It is what taking the cover
|
|
// provider's data back needs, together with forgetting the records.
|
|
func (s *Service) RemoveRemoteAlbumFiles() (int, error) {
|
|
dir := filepath.Join(s.dir, "images", folderAlbumRemote)
|
|
entries, err := os.ReadDir(dir)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return 0, nil
|
|
}
|
|
if err != nil {
|
|
return 0, fmt.Errorf("list the archive covers: %w", err)
|
|
}
|
|
n := 0
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".jpg") {
|
|
continue
|
|
}
|
|
if err := os.Remove(filepath.Join(dir, e.Name())); err != nil {
|
|
return n, fmt.Errorf("remove a cover: %w", err)
|
|
}
|
|
n++
|
|
}
|
|
_ = os.Remove(dir) // goes too when nothing else is in it
|
|
s.tidied.Delete(dir)
|
|
return n, nil
|
|
}
|