fix(jobs): a stop or a panic never wedges a job, a bad download never replaces a good picture

- A panic while counting no longer leaves the job marked as running; a panic in the
  work of one item is a failure of that item, and a panic in the list ends the job
  with a reason.
- A stop while the items are counted is recorded as a stop instead of a 500, and a
  count that fails leaves the status as the previous job left it.
- An artist that lost its MusicBrainz id since it was listed is skipped by the
  pictures job, as the spec says, instead of counting as a failure.
- A picture that cannot be used now (refused download, bytes that are not an image)
  is a failure when a good picture is stored, so a refresh never throws it away.
- The outcomes the artist jobs report share their constants in the jobs package.
- The spec no longer calls each job "the one request" that goes through the library.
- The test that waited with a sleep for the count now waits for a signal.
This commit is contained in:
tgiachi 2026-11-12 10:09:46 +01:00
commit 9323702b1e
12 changed files with 246 additions and 40 deletions

View file

@ -11,8 +11,8 @@ import (
// Outcomes of the descriptions job, as the summary counts them.
const (
OutcomeFound = "found" // a description was found
OutcomeNone = "none" // Wikipedia has none for the artist
OutcomeFound = jobs.Found // a description was found
OutcomeNone = jobs.None // Wikipedia has none for the artist
)
// Lister lists the artists that need their description looked for, so the job does not read

View file

@ -49,7 +49,7 @@ func coversSummary(s *jobs.Summary) any {
// pictures and the descriptions count the same outcomes.
func artistSummary(s *jobs.Summary) any {
return artistSummaryJSON{
Checked: s.Checked, Found: s.Outcomes[images.OutcomeFound], None: s.Outcomes[images.OutcomeNone],
Checked: s.Checked, Found: s.Outcomes[jobs.Found], None: s.Outcomes[jobs.None],
Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped,
}
}

View file

@ -508,7 +508,7 @@ paths:
found one for a week, "no cover" for a day) is skipped; send `refresh=true` to search every
album again.
**This is the one request that goes through the whole library.** With the archive on, it sends
**This goes through the whole library.** With the archive on, it sends
the release group id of every album that has no picture in its files, one after the other and
spaced out. It gives up, and says why in `last_error`, when the provider fails for five albums
in a row, and also when the files cannot be read (an unmounted disk): it never turns that into
@ -550,7 +550,7 @@ paths:
operationId: stopCoversJob
summary: Stop the job that builds the covers
description: |
Admin only. Asks a running job to end after the album it is on; the covers already built stay.
Admin only. Asks a running job to end after the album it is on (its requests to the provider may still finish); the covers already built stay.
It answers the same when no job is running.
security: [{bearerAuth: []}]
responses:
@ -590,7 +590,7 @@ paths:
`refresh=true` to search every artist again. A refresh never replaces a stored picture by a
failure.
**This is the one request that goes through the whole library.** It sends the MusicBrainz id
**This goes through the whole library.** It sends the MusicBrainz id
of every artist, one after the other and spaced out, to MusicBrainz, Wikidata and Commons. It
gives up, and says why in `last_error`, when the providers fail for five artists in a row.
`DELETE /library/artist-images` stops it.
@ -629,7 +629,7 @@ paths:
operationId: stopArtistImagesJob
summary: Stop the job that looks for the artist pictures
description: |
Admin only. Asks a running job to end after the artist it is on; the pictures already found
Admin only. Asks a running job to end after the artist it is on (its requests to the providers may still finish); the pictures already found
stay. It answers the same when no job is running.
security: [{bearerAuth: []}]
responses:
@ -668,7 +668,7 @@ paths:
"no description" for a day; send `refresh=true` to read every artist again. A refresh never
replaces a stored description by a failure.
**This is the one request that goes through the whole library.** It sends the MusicBrainz id
**This goes through the whole library.** It sends the MusicBrainz id
of every artist, one after the other and spaced out, to MusicBrainz, Wikidata and Wikipedia.
It gives up, and says why in `last_error`, when the providers fail for five artists in a row.
`DELETE /library/descriptions` stops it.
@ -707,7 +707,7 @@ paths:
operationId: stopDescriptionsJob
summary: Stop the job that looks for the artist descriptions
description: |
Admin only. Asks a running job to end after the artist it is on; the descriptions already
Admin only. Asks a running job to end after the artist it is on (its requests to the providers may still finish); the descriptions already
found stay. It answers the same when no job is running.
security: [{bearerAuth: []}]
responses:

View file

@ -142,13 +142,13 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
var rec Record
purged := false
for attempt := 0; attempt < 3; attempt++ {
cur, _, err := s.store.Get(ctx, kindArtist, artistID)
cur, had, err := s.store.Get(ctx, kindArtist, artistID)
if err != nil {
return Record{}, fmt.Errorf("read picture record: %w", err)
}
epoch := s.purgeCount()
var pic *Processed
rec, pic, err = s.searchArtist(ctx, artistID)
rec, pic, err = s.searchArtist(ctx, artistID, had && cur.Status == StatusOK)
if errors.Is(err, errPurged) {
continue
}
@ -185,7 +185,10 @@ var errPurged = errors.New("the pictures were taken back while this was being bu
// searchArtist asks the source and, for a picture, prepares it (nil when there is none).
// The files are written by saveArtist, together with the record, so that a purge cannot
// fall between them.
func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Processed, error) {
//
// keepGood says a good picture is stored: a download that cannot be used is then a failure,
// not "no picture", so that the picture is not thrown away for a bad answer.
func (s *Service) searchArtist(ctx context.Context, artistID int64, keepGood bool) (Record, *Processed, error) {
select {
case s.artistSl <- struct{}{}:
defer func() { <-s.artistSl }()
@ -194,7 +197,10 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Pr
}
none := Record{Status: StatusNone, CheckedAt: s.now()}
found, err := s.artists.ArtistImage(ctx, artistID)
bad := fmt.Errorf("%w: what the provider sent cannot be used", ErrSourceUnavailable)
switch {
case errors.Is(err, ErrNoImage) && errors.Is(err, ErrBadPicture) && keepGood:
return Record{}, nil, bad
case errors.Is(err, ErrNoImage):
return none, nil, nil
case err != nil:
@ -202,6 +208,9 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Pr
}
p, err := ProcessLimited(found.Data, MaxArtistPixels)
if err != nil {
if keepGood {
return Record{}, nil, bad
}
return none, nil, nil // what the provider sent is not a usable picture: there is none to show
}
return s.record(p, SourceCommons, found.Credit, none.CheckedAt), &p, nil

View file

@ -9,7 +9,7 @@ import (
)
// OutcomeFound is what the artist pictures job counts for an artist whose picture was found.
const OutcomeFound = "found"
const OutcomeFound = jobs.Found
// ArtistLister lists the artists that need a picture looked for, so the job does not read them all at once.
type ArtistLister interface {
@ -33,7 +33,7 @@ func NewArtistImagesJob(svc *Service, artists ArtistLister) *jobs.Job {
Do: func(ctx context.Context, id int64, refresh bool) (string, error) {
found, err := svc.EnsureArtistImage(ctx, id, refresh)
switch {
case errors.Is(err, ErrArtistNotFound):
case errors.Is(err, ErrArtistNotFound), errors.Is(err, ErrNoID):
return "", jobs.ErrSkip
case errors.Is(err, ErrProviderDisabled):
return "", jobs.Off("the commons or the musicbrainz provider is off")

View file

@ -3,6 +3,7 @@ package images
import (
"context"
"errors"
"fmt"
"image/color"
"sync"
"sync/atomic"
@ -201,3 +202,50 @@ func TestAnArtistThatWentIsSkippedByThePicturesJob(t *testing.T) {
t.Errorf("status = %+v", st)
}
}
// ---- a bad download never replaces a good picture ----
func TestAPictureThatCannotBeUsedNowDoesNotReplaceAGoodOne(t *testing.T) {
bad := map[string]func(r *artistRig){
"the download is refused": func(r *artistRig) { r.src.set(Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture)) },
"the bytes are not a picture": func(r *artistRig) {
r.src.set(Found{Data: []byte("<html>502 Bad Gateway</html>"), Credit: credit}, nil)
},
}
for name, breakIt := range bad {
t.Run(name, func(t *testing.T) {
r := newArtistRig(t)
_, _ = r.svc.EnsureArtistImage(context.Background(), 7, false)
before := r.store.recs["artist/7"]
breakIt(r)
if _, err := r.svc.EnsureArtistImage(context.Background(), 7, true); !errors.Is(err, ErrSourceUnavailable) {
t.Fatalf("err = %v, want ErrSourceUnavailable: the answer is not about the picture", err)
}
if after := r.store.recs["artist/7"]; after != before {
t.Errorf("record = %+v, want it untouched: %+v", after, before)
}
r.get(t, 7, Large) // still served, files included
})
}
}
func TestAPictureThatCannotBeUsedIsNoPictureWhenThereWasNoneBefore(t *testing.T) {
r := newArtistRig(t)
r.src.set(Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture))
if found, err := r.svc.EnsureArtistImage(context.Background(), 7, false); err != nil || found {
t.Errorf("found = %v, err = %v, want no picture and no error", found, err)
}
if rec := r.store.recs["artist/7"]; rec.Status != StatusNone {
t.Errorf("record = %+v", rec)
}
}
func TestAnArtistThatLostItsIdIsSkippedByThePicturesJob(t *testing.T) {
r, job := newPicturesJob(t)
r.src.set(Found{}, ErrNoID)
_ = job.Start(context.Background(), jobs.Options{})
st := waitDone(t, job)
if st.Last.Skipped != 6 || st.Last.Failed != 0 || st.LastError != "" {
t.Errorf("status = %+v", st)
}
}

View file

@ -123,7 +123,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
data, err := c.Commons.Download(ctx, file.ThumbURL)
switch {
case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrUnusable):
return Found{}, ErrNoImage
return Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture)
case err != nil:
return Found{}, failed(ctx, err)
}

View file

@ -226,7 +226,7 @@ func TestARequestThatTheCallerCancelledIsReturnedAsCancelled(t *testing.T) {
func TestAPictureThatCannotBeUsedFromTheHostIsNoPicture(t *testing.T) {
c := newChain()
c.cm.loadErr = commons.ErrUnusable // a redirect or a picture over the limit
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) {
t.Errorf("err = %v, want ErrNoImage", err)
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) || !errors.Is(err, ErrBadPicture) {
t.Errorf("err = %v, want ErrNoImage that says the picture itself was bad", err)
}
}

View file

@ -12,7 +12,7 @@ import (
const (
OutcomeFromFiles = "from_files" // a cover found in the user's own files
OutcomeFromArchive = "from_archive" // a cover from the Cover Art Archive
OutcomeNone = "none" // no cover anywhere
OutcomeNone = jobs.None // no cover anywhere
)
// AlbumLister lists the albums that need a cover built, so the job does not read them all at once.

View file

@ -79,9 +79,13 @@ var (
ErrFilesUnreadable = errors.New("the files of the album could not be read")
// Answers of an ArtistSource.
ErrNoImage = errors.New("there is no free picture of this artist")
ErrProviderDisabled = errors.New("the provider is switched off")
ErrNoID = errors.New("the artist has no MusicBrainz id")
ErrNoImage = errors.New("there is no free picture of this artist")
ErrProviderDisabled = errors.New("the provider is switched off")
ErrNoID = errors.New("the artist has no MusicBrainz id")
// ErrBadPicture is wrapped together with ErrNoImage when what the provider sent could not
// be used (a refused download, bytes that are not a picture): there is no picture to show,
// but that is not an answer about the artist, so it never replaces a picture that was good.
ErrBadPicture = errors.New("the picture could not be used")
ErrArtistNotFound = errors.New("artist not found")
ErrSourceUnavailable = errors.New("the provider could not be reached")
)

View file

@ -34,6 +34,13 @@ func (o off) Is(target error) bool { return target == ErrOff }
// fail the same way, so the job ends, and reason is what the status says about it.
func Off(reason string) error { return off{reason} }
// Outcomes that the jobs for the artists report, so that whoever shows a summary does not
// depend on two packages spelling them the same way.
const (
Found = "found" // something was found for the artist
None = "none" // the provider has nothing for the artist
)
// MaxConsecutiveFailures is how many items in a row may fail before the job gives up.
const MaxConsecutiveFailures = 5
@ -104,7 +111,7 @@ func New(t Task, now func() time.Time) *Job {
// Start begins the job in the background. ctx bounds it, so pass one that lives as long as the
// application, not an HTTP request. It returns ErrRunning when one is already running.
func (j *Job) Start(ctx context.Context, o Options) error {
func (j *Job) Start(ctx context.Context, o Options) (err error) {
now := j.now()
runCtx, cancel := context.WithCancel(ctx)
// The job is marked as running before the items are counted, which can take a while on a
@ -115,22 +122,46 @@ func (j *Job) Start(ctx context.Context, o Options) error {
cancel()
return ErrRunning
}
prev := j.status
j.cancel = cancel
j.cur, j.total = Summary{Outcomes: map[string]int{}}, 0
j.status.Running, j.status.StartedAt = true, now
j.mu.Unlock()
total, err := j.task.Count(runCtx, o.Refresh, now)
if err != nil {
j.mu.Lock()
j.status.Running = false
j.mu.Unlock()
launched := false
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("count the %s: panic: %v", j.task.Noun, r)
}
if launched {
return
}
cancel()
return fmt.Errorf("count the %s: %w", j.task.Noun, err)
j.mu.Lock()
defer j.mu.Unlock()
if runCtx.Err() != nil && err == nil {
// Stopped while counting: that is how this job ended, and the status says so.
j.status.Running = false
j.status.LastFinishedAt = j.now()
j.status.Last = &Summary{Outcomes: map[string]int{}, Stopped: true}
j.status.LastError = ""
return
}
// It never started: the status stays as the last job left it.
j.status = prev
}()
total, cerr := j.task.Count(runCtx, o.Refresh, now)
if cerr != nil {
if runCtx.Err() != nil {
return nil
}
return fmt.Errorf("count the %s: %w", j.task.Noun, cerr)
}
j.mu.Lock()
j.total = total
j.mu.Unlock()
launched = true
go j.run(runCtx, cancel, o, now)
return nil
}
@ -176,11 +207,35 @@ func (j *Job) count(f func(*Summary)) {
f(&j.cur)
}
// do runs the work on one item; a panic in it is a failure of that item, not a crash.
func (j *Job) do(ctx context.Context, id int64, refresh bool) (outcome string, err error) {
defer func() {
if r := recover(); r != nil {
outcome, err = "", fmt.Errorf("panic: %v", r)
}
}()
return j.task.Do(ctx, id, refresh)
}
func (j *Job) run(ctx context.Context, cancel context.CancelFunc, o Options, started time.Time) {
defer cancel() // release the context when it ends by itself
reason := ""
failures := 0
stopped := false
// However it ends, the job must stop running: whatever panicked in the list ends it with a reason.
defer func() {
if r := recover(); r != nil {
reason = fmt.Sprintf("the job panicked: %v", r)
}
j.mu.Lock()
defer j.mu.Unlock()
sum := copySummary(j.cur)
sum.Stopped = stopped
j.status.Running = false
j.status.LastFinishedAt = j.now()
j.status.Last = &sum
j.status.LastError = reason
}()
loop:
for after := int64(0); ; {
ids, err := j.task.List(ctx, after, page, o.Refresh, started)
@ -201,7 +256,7 @@ loop:
stopped = true
break loop
}
outcome, err := j.task.Do(ctx, id, o.Refresh)
outcome, err := j.do(ctx, id, o.Refresh)
switch {
case err != nil && ctx.Err() != nil:
stopped = true
@ -225,15 +280,6 @@ loop:
}
}
}
j.mu.Lock()
defer j.mu.Unlock()
sum := copySummary(j.cur)
sum.Stopped = stopped
j.status.Running = false
j.status.LastFinishedAt = j.now()
j.status.Last = &sum
j.status.LastError = reason
}
// Runner is a Job started with the context of the application, which is what the API needs:

View file

@ -15,10 +15,12 @@ type rig struct {
mu sync.Mutex
ids []int64
countErr error
countPanic any
listErr error
failOnPage int // fail from this page on (1 = the first); 0 = never
pageSize int // 0 = as asked
countGate chan struct{} // when set, Count waits for it to be closed
counting chan struct{} // when set, Count signals here that it was entered
refresh []bool // the refresh flag of every list call
pages int
@ -41,11 +43,17 @@ func newRig() *rig {
func (r *rig) count(context.Context, bool, time.Time) (int, error) {
r.mu.Lock()
gate, n, err := r.countGate, len(r.ids), r.countErr
gate, entered, n, err, boom := r.countGate, r.counting, len(r.ids), r.countErr, r.countPanic
r.mu.Unlock()
if entered != nil {
entered <- struct{}{}
}
if gate != nil {
<-gate
}
if boom != nil {
panic(boom)
}
return n, err
}
@ -310,9 +318,10 @@ func TestAProblemListingDuringTheJobEndsItWithItsReason(t *testing.T) {
func TestStartingDoesNotHoldTheStatusWhileTheItemsAreCounted(t *testing.T) {
r := newRig()
r.countGate = make(chan struct{})
r.counting = make(chan struct{}, 2)
started := make(chan error, 1)
go func() { started <- r.job.Start(context.Background(), Options{}) }()
time.Sleep(40 * time.Millisecond)
<-r.counting // the count is running now
done := make(chan Status, 1)
go func() { done <- r.job.Status() }()
select {
@ -386,3 +395,93 @@ func TestARunnerStartsStopsAndReadsAJobWithTheContextOfTheApplication(t *testing
t.Error("the runner did not pass the refresh option")
}
}
func TestAStopWhileTheItemsAreCountedIsReportedAsAStop(t *testing.T) {
r := newRig()
r.countGate = make(chan struct{})
r.counting = make(chan struct{}, 2)
var calls int32
r.job = New(Task{Noun: "things", List: r.list, Do: r.run,
Count: func(ctx context.Context, refresh bool, now time.Time) (int, error) {
n, err := r.count(ctx, refresh, now)
if atomic.AddInt32(&calls, 1) > 1 {
return n, err
}
<-ctx.Done() // a count on a big library notices that it was stopped
if err == nil {
err = ctx.Err()
}
return n, err
}}, time.Now)
started := make(chan error, 1)
go func() { started <- r.job.Start(context.Background(), Options{}) }()
<-r.counting
if !r.job.Stop() {
t.Fatal("Stop says there was nothing to stop")
}
close(r.countGate)
if err := <-started; err != nil {
t.Fatalf("Start = %v: a stop is not an error", err)
}
st := r.job.Status()
if st.Running || st.Last == nil || !st.Last.Stopped || st.LastFinishedAt.IsZero() {
t.Errorf("status = %+v, want a finished job that was stopped", st)
}
if err := r.job.Start(context.Background(), Options{}); err != nil {
t.Errorf("it cannot start again after the stop: %v", err)
}
waitDone(t, r.job)
}
func TestACountThatFailsLeavesTheStatusAsItWas(t *testing.T) {
r := newRig()
first := r.start(t)
r.countErr = errors.New("database is locked")
if err := r.job.Start(context.Background(), Options{}); err == nil {
t.Fatal("the job started")
}
st := r.job.Status()
if st.Running || !st.StartedAt.Equal(first.StartedAt) || st.Last == nil || st.Last.Checked != 6 {
t.Errorf("status = %+v, want the one of the job that ran before", st)
}
}
func TestACountThatPanicsDoesNotLeaveTheJobRunning(t *testing.T) {
r := newRig()
r.countPanic = "boom"
if err := r.job.Start(context.Background(), Options{}); err == nil {
t.Fatal("the job started although counting panicked")
}
if r.job.Status().Running {
t.Fatal("the job says it runs after counting panicked")
}
r.countPanic = nil
if err := r.job.Start(context.Background(), Options{}); err != nil {
t.Errorf("it cannot start again: %v", err)
}
waitDone(t, r.job)
}
func TestAPanicInTheWorkIsAFailureOfThatItemNotACrash(t *testing.T) {
r := newRig()
r.do = func(id int64) (string, error) {
if id == 3 {
panic("boom")
}
return "found", nil
}
st := r.start(t)
if st.Last.Checked != 6 || st.Last.Failed != 1 || st.Last.Outcomes["found"] != 5 || st.LastError != "" {
t.Errorf("status = %+v", st)
}
}
func TestAPanicInTheListEndsTheJobWithAReasonAndFreesIt(t *testing.T) {
r := newRig()
r.job = New(Task{Noun: "things", Count: r.count, Do: r.run,
List: func(context.Context, int64, int, bool, time.Time) ([]int64, error) { panic("boom") }}, time.Now)
st := r.start(t)
if st.Last == nil || st.LastError == "" || st.Running {
t.Errorf("status = %+v", st)
}
}