From 9323702b1eaa56d9d137c4d666d7baff0c520b30 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 10:09:46 +0100 Subject: [PATCH] fix(jobs): a stop or a panic never wedges a job, a bad download never replaces a good picture - A panic while counting no longer leaves the job marked as running; a panic in the work of one item is a failure of that item, and a panic in the list ends the job with a reason. - A stop while the items are counted is recorded as a stop instead of a 500, and a count that fails leaves the status as the previous job left it. - An artist that lost its MusicBrainz id since it was listed is skipped by the pictures job, as the spec says, instead of counting as a failure. - A picture that cannot be used now (refused download, bytes that are not an image) is a failure when a good picture is stored, so a refresh never throws it away. - The outcomes the artist jobs report share their constants in the jobs package. - The spec no longer calls each job "the one request" that goes through the library. - The test that waited with a sleep for the count now waits for a signal. --- internal/about/job.go | 4 +- internal/api/http/handlers_jobs.go | 2 +- internal/api/openapi/openapi.yaml | 12 +-- internal/images/artist.go | 15 +++- internal/images/artist_job.go | 4 +- internal/images/artist_job_test.go | 48 ++++++++++++ internal/images/commons_source.go | 2 +- internal/images/commons_source_test.go | 4 +- internal/images/job.go | 2 +- internal/images/service.go | 10 ++- internal/jobs/jobs.go | 80 +++++++++++++++---- internal/jobs/jobs_test.go | 103 ++++++++++++++++++++++++- 12 files changed, 246 insertions(+), 40 deletions(-) diff --git a/internal/about/job.go b/internal/about/job.go index 6d23040..1b56fda 100644 --- a/internal/about/job.go +++ b/internal/about/job.go @@ -11,8 +11,8 @@ import ( // Outcomes of the descriptions job, as the summary counts them. const ( - OutcomeFound = "found" // a description was found - OutcomeNone = "none" // Wikipedia has none for the artist + OutcomeFound = jobs.Found // a description was found + OutcomeNone = jobs.None // Wikipedia has none for the artist ) // Lister lists the artists that need their description looked for, so the job does not read diff --git a/internal/api/http/handlers_jobs.go b/internal/api/http/handlers_jobs.go index e815874..dcc832a 100644 --- a/internal/api/http/handlers_jobs.go +++ b/internal/api/http/handlers_jobs.go @@ -49,7 +49,7 @@ func coversSummary(s *jobs.Summary) any { // pictures and the descriptions count the same outcomes. func artistSummary(s *jobs.Summary) any { return artistSummaryJSON{ - Checked: s.Checked, Found: s.Outcomes[images.OutcomeFound], None: s.Outcomes[images.OutcomeNone], + Checked: s.Checked, Found: s.Outcomes[jobs.Found], None: s.Outcomes[jobs.None], Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped, } } diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 7714f9f..58e733b 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -508,7 +508,7 @@ paths: found one for a week, "no cover" for a day) is skipped; send `refresh=true` to search every album again. - **This is the one request that goes through the whole library.** With the archive on, it sends + **This goes through the whole library.** With the archive on, it sends the release group id of every album that has no picture in its files, one after the other and spaced out. It gives up, and says why in `last_error`, when the provider fails for five albums in a row, and also when the files cannot be read (an unmounted disk): it never turns that into @@ -550,7 +550,7 @@ paths: operationId: stopCoversJob summary: Stop the job that builds the covers description: | - Admin only. Asks a running job to end after the album it is on; the covers already built stay. + Admin only. Asks a running job to end after the album it is on (its requests to the provider may still finish); the covers already built stay. It answers the same when no job is running. security: [{bearerAuth: []}] responses: @@ -590,7 +590,7 @@ paths: `refresh=true` to search every artist again. A refresh never replaces a stored picture by a failure. - **This is the one request that goes through the whole library.** It sends the MusicBrainz id + **This goes through the whole library.** It sends the MusicBrainz id of every artist, one after the other and spaced out, to MusicBrainz, Wikidata and Commons. It gives up, and says why in `last_error`, when the providers fail for five artists in a row. `DELETE /library/artist-images` stops it. @@ -629,7 +629,7 @@ paths: operationId: stopArtistImagesJob summary: Stop the job that looks for the artist pictures description: | - Admin only. Asks a running job to end after the artist it is on; the pictures already found + Admin only. Asks a running job to end after the artist it is on (its requests to the providers may still finish); the pictures already found stay. It answers the same when no job is running. security: [{bearerAuth: []}] responses: @@ -668,7 +668,7 @@ paths: "no description" for a day; send `refresh=true` to read every artist again. A refresh never replaces a stored description by a failure. - **This is the one request that goes through the whole library.** It sends the MusicBrainz id + **This goes through the whole library.** It sends the MusicBrainz id of every artist, one after the other and spaced out, to MusicBrainz, Wikidata and Wikipedia. It gives up, and says why in `last_error`, when the providers fail for five artists in a row. `DELETE /library/descriptions` stops it. @@ -707,7 +707,7 @@ paths: operationId: stopDescriptionsJob summary: Stop the job that looks for the artist descriptions description: | - Admin only. Asks a running job to end after the artist it is on; the descriptions already + Admin only. Asks a running job to end after the artist it is on (its requests to the providers may still finish); the descriptions already found stay. It answers the same when no job is running. security: [{bearerAuth: []}] responses: diff --git a/internal/images/artist.go b/internal/images/artist.go index e145632..716c564 100644 --- a/internal/images/artist.go +++ b/internal/images/artist.go @@ -142,13 +142,13 @@ func (s *Service) buildArtist(artistID int64) (Record, error) { var rec Record purged := false for attempt := 0; attempt < 3; attempt++ { - cur, _, err := s.store.Get(ctx, kindArtist, artistID) + cur, had, err := s.store.Get(ctx, kindArtist, artistID) if err != nil { return Record{}, fmt.Errorf("read picture record: %w", err) } epoch := s.purgeCount() var pic *Processed - rec, pic, err = s.searchArtist(ctx, artistID) + rec, pic, err = s.searchArtist(ctx, artistID, had && cur.Status == StatusOK) if errors.Is(err, errPurged) { continue } @@ -185,7 +185,10 @@ var errPurged = errors.New("the pictures were taken back while this was being bu // searchArtist asks the source and, for a picture, prepares it (nil when there is none). // The files are written by saveArtist, together with the record, so that a purge cannot // fall between them. -func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Processed, error) { +// +// keepGood says a good picture is stored: a download that cannot be used is then a failure, +// not "no picture", so that the picture is not thrown away for a bad answer. +func (s *Service) searchArtist(ctx context.Context, artistID int64, keepGood bool) (Record, *Processed, error) { select { case s.artistSl <- struct{}{}: defer func() { <-s.artistSl }() @@ -194,7 +197,10 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Pr } none := Record{Status: StatusNone, CheckedAt: s.now()} found, err := s.artists.ArtistImage(ctx, artistID) + bad := fmt.Errorf("%w: what the provider sent cannot be used", ErrSourceUnavailable) switch { + case errors.Is(err, ErrNoImage) && errors.Is(err, ErrBadPicture) && keepGood: + return Record{}, nil, bad case errors.Is(err, ErrNoImage): return none, nil, nil case err != nil: @@ -202,6 +208,9 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Pr } p, err := ProcessLimited(found.Data, MaxArtistPixels) if err != nil { + if keepGood { + return Record{}, nil, bad + } return none, nil, nil // what the provider sent is not a usable picture: there is none to show } return s.record(p, SourceCommons, found.Credit, none.CheckedAt), &p, nil diff --git a/internal/images/artist_job.go b/internal/images/artist_job.go index 7c51969..09ec6ed 100644 --- a/internal/images/artist_job.go +++ b/internal/images/artist_job.go @@ -9,7 +9,7 @@ import ( ) // OutcomeFound is what the artist pictures job counts for an artist whose picture was found. -const OutcomeFound = "found" +const OutcomeFound = jobs.Found // ArtistLister lists the artists that need a picture looked for, so the job does not read them all at once. type ArtistLister interface { @@ -33,7 +33,7 @@ func NewArtistImagesJob(svc *Service, artists ArtistLister) *jobs.Job { Do: func(ctx context.Context, id int64, refresh bool) (string, error) { found, err := svc.EnsureArtistImage(ctx, id, refresh) switch { - case errors.Is(err, ErrArtistNotFound): + case errors.Is(err, ErrArtistNotFound), errors.Is(err, ErrNoID): return "", jobs.ErrSkip case errors.Is(err, ErrProviderDisabled): return "", jobs.Off("the commons or the musicbrainz provider is off") diff --git a/internal/images/artist_job_test.go b/internal/images/artist_job_test.go index f4cb73f..58f8d1f 100644 --- a/internal/images/artist_job_test.go +++ b/internal/images/artist_job_test.go @@ -3,6 +3,7 @@ package images import ( "context" "errors" + "fmt" "image/color" "sync" "sync/atomic" @@ -201,3 +202,50 @@ func TestAnArtistThatWentIsSkippedByThePicturesJob(t *testing.T) { t.Errorf("status = %+v", st) } } + +// ---- a bad download never replaces a good picture ---- + +func TestAPictureThatCannotBeUsedNowDoesNotReplaceAGoodOne(t *testing.T) { + bad := map[string]func(r *artistRig){ + "the download is refused": func(r *artistRig) { r.src.set(Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture)) }, + "the bytes are not a picture": func(r *artistRig) { + r.src.set(Found{Data: []byte("502 Bad Gateway"), Credit: credit}, nil) + }, + } + for name, breakIt := range bad { + t.Run(name, func(t *testing.T) { + r := newArtistRig(t) + _, _ = r.svc.EnsureArtistImage(context.Background(), 7, false) + before := r.store.recs["artist/7"] + breakIt(r) + if _, err := r.svc.EnsureArtistImage(context.Background(), 7, true); !errors.Is(err, ErrSourceUnavailable) { + t.Fatalf("err = %v, want ErrSourceUnavailable: the answer is not about the picture", err) + } + if after := r.store.recs["artist/7"]; after != before { + t.Errorf("record = %+v, want it untouched: %+v", after, before) + } + r.get(t, 7, Large) // still served, files included + }) + } +} + +func TestAPictureThatCannotBeUsedIsNoPictureWhenThereWasNoneBefore(t *testing.T) { + r := newArtistRig(t) + r.src.set(Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture)) + if found, err := r.svc.EnsureArtistImage(context.Background(), 7, false); err != nil || found { + t.Errorf("found = %v, err = %v, want no picture and no error", found, err) + } + if rec := r.store.recs["artist/7"]; rec.Status != StatusNone { + t.Errorf("record = %+v", rec) + } +} + +func TestAnArtistThatLostItsIdIsSkippedByThePicturesJob(t *testing.T) { + r, job := newPicturesJob(t) + r.src.set(Found{}, ErrNoID) + _ = job.Start(context.Background(), jobs.Options{}) + st := waitDone(t, job) + if st.Last.Skipped != 6 || st.Last.Failed != 0 || st.LastError != "" { + t.Errorf("status = %+v", st) + } +} diff --git a/internal/images/commons_source.go b/internal/images/commons_source.go index d75c416..57a6520 100644 --- a/internal/images/commons_source.go +++ b/internal/images/commons_source.go @@ -123,7 +123,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found, data, err := c.Commons.Download(ctx, file.ThumbURL) switch { case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrUnusable): - return Found{}, ErrNoImage + return Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture) case err != nil: return Found{}, failed(ctx, err) } diff --git a/internal/images/commons_source_test.go b/internal/images/commons_source_test.go index 529faa1..142fba9 100644 --- a/internal/images/commons_source_test.go +++ b/internal/images/commons_source_test.go @@ -226,7 +226,7 @@ func TestARequestThatTheCallerCancelledIsReturnedAsCancelled(t *testing.T) { func TestAPictureThatCannotBeUsedFromTheHostIsNoPicture(t *testing.T) { c := newChain() c.cm.loadErr = commons.ErrUnusable // a redirect or a picture over the limit - if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) { - t.Errorf("err = %v, want ErrNoImage", err) + if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) || !errors.Is(err, ErrBadPicture) { + t.Errorf("err = %v, want ErrNoImage that says the picture itself was bad", err) } } diff --git a/internal/images/job.go b/internal/images/job.go index d942a72..f521652 100644 --- a/internal/images/job.go +++ b/internal/images/job.go @@ -12,7 +12,7 @@ import ( const ( OutcomeFromFiles = "from_files" // a cover found in the user's own files OutcomeFromArchive = "from_archive" // a cover from the Cover Art Archive - OutcomeNone = "none" // no cover anywhere + OutcomeNone = jobs.None // no cover anywhere ) // AlbumLister lists the albums that need a cover built, so the job does not read them all at once. diff --git a/internal/images/service.go b/internal/images/service.go index 7bb64fa..540145e 100644 --- a/internal/images/service.go +++ b/internal/images/service.go @@ -79,9 +79,13 @@ var ( ErrFilesUnreadable = errors.New("the files of the album could not be read") // Answers of an ArtistSource. - ErrNoImage = errors.New("there is no free picture of this artist") - ErrProviderDisabled = errors.New("the provider is switched off") - ErrNoID = errors.New("the artist has no MusicBrainz id") + ErrNoImage = errors.New("there is no free picture of this artist") + ErrProviderDisabled = errors.New("the provider is switched off") + ErrNoID = errors.New("the artist has no MusicBrainz id") + // ErrBadPicture is wrapped together with ErrNoImage when what the provider sent could not + // be used (a refused download, bytes that are not a picture): there is no picture to show, + // but that is not an answer about the artist, so it never replaces a picture that was good. + ErrBadPicture = errors.New("the picture could not be used") ErrArtistNotFound = errors.New("artist not found") ErrSourceUnavailable = errors.New("the provider could not be reached") ) diff --git a/internal/jobs/jobs.go b/internal/jobs/jobs.go index 218c041..349ff03 100644 --- a/internal/jobs/jobs.go +++ b/internal/jobs/jobs.go @@ -34,6 +34,13 @@ func (o off) Is(target error) bool { return target == ErrOff } // fail the same way, so the job ends, and reason is what the status says about it. func Off(reason string) error { return off{reason} } +// Outcomes that the jobs for the artists report, so that whoever shows a summary does not +// depend on two packages spelling them the same way. +const ( + Found = "found" // something was found for the artist + None = "none" // the provider has nothing for the artist +) + // MaxConsecutiveFailures is how many items in a row may fail before the job gives up. const MaxConsecutiveFailures = 5 @@ -104,7 +111,7 @@ func New(t Task, now func() time.Time) *Job { // Start begins the job in the background. ctx bounds it, so pass one that lives as long as the // application, not an HTTP request. It returns ErrRunning when one is already running. -func (j *Job) Start(ctx context.Context, o Options) error { +func (j *Job) Start(ctx context.Context, o Options) (err error) { now := j.now() runCtx, cancel := context.WithCancel(ctx) // The job is marked as running before the items are counted, which can take a while on a @@ -115,22 +122,46 @@ func (j *Job) Start(ctx context.Context, o Options) error { cancel() return ErrRunning } + prev := j.status j.cancel = cancel j.cur, j.total = Summary{Outcomes: map[string]int{}}, 0 j.status.Running, j.status.StartedAt = true, now j.mu.Unlock() - total, err := j.task.Count(runCtx, o.Refresh, now) - if err != nil { - j.mu.Lock() - j.status.Running = false - j.mu.Unlock() + launched := false + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("count the %s: panic: %v", j.task.Noun, r) + } + if launched { + return + } cancel() - return fmt.Errorf("count the %s: %w", j.task.Noun, err) + j.mu.Lock() + defer j.mu.Unlock() + if runCtx.Err() != nil && err == nil { + // Stopped while counting: that is how this job ended, and the status says so. + j.status.Running = false + j.status.LastFinishedAt = j.now() + j.status.Last = &Summary{Outcomes: map[string]int{}, Stopped: true} + j.status.LastError = "" + return + } + // It never started: the status stays as the last job left it. + j.status = prev + }() + + total, cerr := j.task.Count(runCtx, o.Refresh, now) + if cerr != nil { + if runCtx.Err() != nil { + return nil + } + return fmt.Errorf("count the %s: %w", j.task.Noun, cerr) } j.mu.Lock() j.total = total j.mu.Unlock() + launched = true go j.run(runCtx, cancel, o, now) return nil } @@ -176,11 +207,35 @@ func (j *Job) count(f func(*Summary)) { f(&j.cur) } +// do runs the work on one item; a panic in it is a failure of that item, not a crash. +func (j *Job) do(ctx context.Context, id int64, refresh bool) (outcome string, err error) { + defer func() { + if r := recover(); r != nil { + outcome, err = "", fmt.Errorf("panic: %v", r) + } + }() + return j.task.Do(ctx, id, refresh) +} + func (j *Job) run(ctx context.Context, cancel context.CancelFunc, o Options, started time.Time) { defer cancel() // release the context when it ends by itself reason := "" failures := 0 stopped := false + // However it ends, the job must stop running: whatever panicked in the list ends it with a reason. + defer func() { + if r := recover(); r != nil { + reason = fmt.Sprintf("the job panicked: %v", r) + } + j.mu.Lock() + defer j.mu.Unlock() + sum := copySummary(j.cur) + sum.Stopped = stopped + j.status.Running = false + j.status.LastFinishedAt = j.now() + j.status.Last = &sum + j.status.LastError = reason + }() loop: for after := int64(0); ; { ids, err := j.task.List(ctx, after, page, o.Refresh, started) @@ -201,7 +256,7 @@ loop: stopped = true break loop } - outcome, err := j.task.Do(ctx, id, o.Refresh) + outcome, err := j.do(ctx, id, o.Refresh) switch { case err != nil && ctx.Err() != nil: stopped = true @@ -225,15 +280,6 @@ loop: } } } - - j.mu.Lock() - defer j.mu.Unlock() - sum := copySummary(j.cur) - sum.Stopped = stopped - j.status.Running = false - j.status.LastFinishedAt = j.now() - j.status.Last = &sum - j.status.LastError = reason } // Runner is a Job started with the context of the application, which is what the API needs: diff --git a/internal/jobs/jobs_test.go b/internal/jobs/jobs_test.go index 22853e3..4358b60 100644 --- a/internal/jobs/jobs_test.go +++ b/internal/jobs/jobs_test.go @@ -15,10 +15,12 @@ type rig struct { mu sync.Mutex ids []int64 countErr error + countPanic any listErr error failOnPage int // fail from this page on (1 = the first); 0 = never pageSize int // 0 = as asked countGate chan struct{} // when set, Count waits for it to be closed + counting chan struct{} // when set, Count signals here that it was entered refresh []bool // the refresh flag of every list call pages int @@ -41,11 +43,17 @@ func newRig() *rig { func (r *rig) count(context.Context, bool, time.Time) (int, error) { r.mu.Lock() - gate, n, err := r.countGate, len(r.ids), r.countErr + gate, entered, n, err, boom := r.countGate, r.counting, len(r.ids), r.countErr, r.countPanic r.mu.Unlock() + if entered != nil { + entered <- struct{}{} + } if gate != nil { <-gate } + if boom != nil { + panic(boom) + } return n, err } @@ -310,9 +318,10 @@ func TestAProblemListingDuringTheJobEndsItWithItsReason(t *testing.T) { func TestStartingDoesNotHoldTheStatusWhileTheItemsAreCounted(t *testing.T) { r := newRig() r.countGate = make(chan struct{}) + r.counting = make(chan struct{}, 2) started := make(chan error, 1) go func() { started <- r.job.Start(context.Background(), Options{}) }() - time.Sleep(40 * time.Millisecond) + <-r.counting // the count is running now done := make(chan Status, 1) go func() { done <- r.job.Status() }() select { @@ -386,3 +395,93 @@ func TestARunnerStartsStopsAndReadsAJobWithTheContextOfTheApplication(t *testing t.Error("the runner did not pass the refresh option") } } + +func TestAStopWhileTheItemsAreCountedIsReportedAsAStop(t *testing.T) { + r := newRig() + r.countGate = make(chan struct{}) + r.counting = make(chan struct{}, 2) + var calls int32 + r.job = New(Task{Noun: "things", List: r.list, Do: r.run, + Count: func(ctx context.Context, refresh bool, now time.Time) (int, error) { + n, err := r.count(ctx, refresh, now) + if atomic.AddInt32(&calls, 1) > 1 { + return n, err + } + <-ctx.Done() // a count on a big library notices that it was stopped + if err == nil { + err = ctx.Err() + } + return n, err + }}, time.Now) + started := make(chan error, 1) + go func() { started <- r.job.Start(context.Background(), Options{}) }() + <-r.counting + if !r.job.Stop() { + t.Fatal("Stop says there was nothing to stop") + } + close(r.countGate) + if err := <-started; err != nil { + t.Fatalf("Start = %v: a stop is not an error", err) + } + st := r.job.Status() + if st.Running || st.Last == nil || !st.Last.Stopped || st.LastFinishedAt.IsZero() { + t.Errorf("status = %+v, want a finished job that was stopped", st) + } + if err := r.job.Start(context.Background(), Options{}); err != nil { + t.Errorf("it cannot start again after the stop: %v", err) + } + waitDone(t, r.job) +} + +func TestACountThatFailsLeavesTheStatusAsItWas(t *testing.T) { + r := newRig() + first := r.start(t) + r.countErr = errors.New("database is locked") + if err := r.job.Start(context.Background(), Options{}); err == nil { + t.Fatal("the job started") + } + st := r.job.Status() + if st.Running || !st.StartedAt.Equal(first.StartedAt) || st.Last == nil || st.Last.Checked != 6 { + t.Errorf("status = %+v, want the one of the job that ran before", st) + } +} + +func TestACountThatPanicsDoesNotLeaveTheJobRunning(t *testing.T) { + r := newRig() + r.countPanic = "boom" + if err := r.job.Start(context.Background(), Options{}); err == nil { + t.Fatal("the job started although counting panicked") + } + if r.job.Status().Running { + t.Fatal("the job says it runs after counting panicked") + } + r.countPanic = nil + if err := r.job.Start(context.Background(), Options{}); err != nil { + t.Errorf("it cannot start again: %v", err) + } + waitDone(t, r.job) +} + +func TestAPanicInTheWorkIsAFailureOfThatItemNotACrash(t *testing.T) { + r := newRig() + r.do = func(id int64) (string, error) { + if id == 3 { + panic("boom") + } + return "found", nil + } + st := r.start(t) + if st.Last.Checked != 6 || st.Last.Failed != 1 || st.Last.Outcomes["found"] != 5 || st.LastError != "" { + t.Errorf("status = %+v", st) + } +} + +func TestAPanicInTheListEndsTheJobWithAReasonAndFreesIt(t *testing.T) { + r := newRig() + r.job = New(Task{Noun: "things", Count: r.count, Do: r.run, + List: func(context.Context, int64, int, bool, time.Time) ([]int64, error) { panic("boom") }}, time.Now) + st := r.start(t) + if st.Last == nil || st.LastError == "" || st.Running { + t.Errorf("status = %+v", st) + } +}