sonda/internal/about/about.go
tgiachi e3843ad7d6 feat(jobs): background jobs for artist photos and descriptions
Add one engine, internal/jobs, that runs the jobs which go through the whole
library: start, stop, progress, one at a time, give up after five failures in
a row, end at once when a provider is off.

- The covers job now runs on the engine; its API is unchanged.
- POST/GET/DELETE /library/artist-images looks for the picture of every
  artist of the library that has a MusicBrainz id.
- POST/GET/DELETE /library/descriptions does the same for the Wikipedia
  description.
- ?refresh=true searches everything again and never replaces a stored result
  by a failure; artists without an id are never touched, nothing is looked up
  by name.
- images.Service.EnsureArtistImage and about.Service.Ensure do the work for one
  artist; the lists come from ImageRepo and DiscoveryRepo.
- jobs.Runner binds a job to the application context, replacing the two
  hand-written controls.
- OpenAPI, contract test and README describe the new endpoints and say that
  they are bulk requests.
2026-11-12 09:58:45 +01:00

318 lines
11 KiB
Go

// Package about tells what an artist is, with a short text from Wikipedia.
//
// The way to the text is an identifier chain, never a search by name: the artist's
// MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the
// Wikipedia page that Wikidata names in the user's languages. Only the id is
// trusted: the page is used only if it says it is about that same item.
//
// Like the other provider data, it is asked for only when the admin switched the
// provider on and someone asks, kept in the database (30 days; a day when there was
// nothing to find), served stale when Wikipedia fails, and removed with the provider's
// data. The text is CC BY-SA 4.0, so the page, its link and the licence are kept and
// returned with it.
package about
import (
"context"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"go.uber.org/zap"
"golang.org/x/sync/singleflight"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/providers"
)
// Provider is the provider the descriptions come from.
const Provider = providers.Wikipedia
// What has to be shown with the text.
const (
Attribution = "Text from Wikipedia (https://www.wikipedia.org), available under the CC BY-SA 4.0 licence."
LicenseName = "CC BY-SA 4.0"
LicenseURL = "https://creativecommons.org/licenses/by-sa/4.0/"
)
// How long what was read is believed. A description changes slowly; "nothing found"
// is remembered shortly, in case a link was added since.
const (
PositiveTTL = 30 * 24 * time.Hour
NegativeTTL = 24 * time.Hour
)
// MaxTextLen is the longest text kept, in characters. A summary is a paragraph.
const MaxTextLen = 4000
// fetchTimeout bounds a shared read, which does not depend on whoever started it
// still waiting. It covers up to three requests, the first of them to MusicBrainz at
// one a second.
const fetchTimeout = 45 * time.Second
// Errors a Source returns.
var (
// ErrSourceNotFound means the provider does not know what was asked: an unknown
// artist, item or page. It is an answer, not a failure.
ErrSourceNotFound = errors.New("the provider does not know this")
// ErrSourceUnavailable stands for any failure to get an answer.
ErrSourceUnavailable = errors.New("the provider could not be reached")
)
// Description is what Wikipedia says about an artist, with where it comes from.
type Description struct {
Text string // the first paragraph of the article, plain text
Short string // one line, like "English rock band"; may be empty
Language string // of the page: "it", "en"
Title string // of the page
URL string // of the page, on Wikipedia
}
// Snapshot is what is stored for one artist. Fetched is false when the provider was
// never asked; None when it was and there was nothing to find.
type Snapshot struct {
Fetched bool
None bool
RetrievedAt time.Time
Description Description
}
// Result is the answer to Description. RetrievedAt is zero when nothing is known; Stale
// means the description is older than it should be because Wikipedia could not be asked.
type Result struct {
Status string // discovery.StatusOK, StatusDisabled, StatusError or StatusNoID
Stale bool
RetrievedAt time.Time
Description *Description // nil when there is none
}
// Store keeps the descriptions. Artist returns discovery.ErrNotFound for an id the
// catalog does not know, and so does SaveDescription for an artist removed meanwhile.
type Store interface {
Artist(ctx context.Context, id int64) (discovery.Artist, error)
Description(ctx context.Context, id int64) (Snapshot, error)
// SaveDescription replaces what is stored; a nil description records that there is
// none. wikidataID is the item MusicBrainz linked, or "".
SaveDescription(ctx context.Context, id int64, at time.Time, wikidataID string, d *Description) error
}
// Summary is what a Wikipedia page says about itself.
type Summary struct {
Type string
Title string
Lang string
Extract string
Description string
URL string
WikibaseItem string
}
// Source follows the chain of identifiers. Every method returns ErrSourceNotFound for
// what the provider does not know, and any other error for a failure.
type Source interface {
// WikidataID returns the Wikidata item MusicBrainz links to the artist, or "".
WikidataID(ctx context.Context, musicbrainzID string) (string, error)
// Pages returns the title of the page of the item in each of the languages that has
// one, and the item they belong to: the same, or the one it was merged into.
Pages(ctx context.Context, item string, languages []string) (string, map[string]string, error)
Summary(ctx context.Context, language, title string) (Summary, error)
}
// Service answers "what is this artist".
type Service struct {
store Store
source Source
settings discovery.Settings
languages []string
log *zap.Logger
now func() time.Time
flight singleflight.Group
}
// NewService builds a Service. languages are the Wikipedia languages to try, in order.
func NewService(store Store, source Source, settings discovery.Settings, languages []string, log *zap.Logger) *Service {
if log == nil {
log = zap.NewNop()
}
return &Service{store: store, source: source, settings: settings, languages: languages, log: log, now: time.Now}
}
// Description returns the description of an artist. A provider that is off, cannot be
// asked or fails is not an error: the answer says so and carries what was stored.
func (s *Service) Description(ctx context.Context, artistID int64) (Result, error) {
return s.describe(ctx, artistID, false)
}
// Ensure is Description for the job that goes through the library: with refresh it reads
// again even a description that is still good. Whatever goes wrong is in the status of the
// answer, which then still carries what was stored: a refresh never replaces a description
// by a failure.
func (s *Service) Ensure(ctx context.Context, artistID int64, refresh bool) (Result, error) {
return s.describe(ctx, artistID, refresh)
}
func (s *Service) describe(ctx context.Context, artistID int64, force bool) (Result, error) {
if artistID <= 0 {
return Result{}, discovery.ErrNotFound
}
artist, err := s.store.Artist(ctx, artistID)
if err != nil {
return Result{}, err
}
snap, err := s.store.Description(ctx, artistID)
if err != nil {
return Result{}, err
}
fresh := !force && snap.Fetched && s.fresh(snap)
active, err := s.active(ctx)
if err != nil {
return Result{}, err
}
switch {
case !active:
return result(snap, discovery.StatusDisabled, snap.Fetched && !fresh), nil
case fresh:
return result(snap, discovery.StatusOK, false), nil
}
if artist.MBID == "" { // Sonda never looks an id up by name
return result(snap, discovery.StatusNoID, snap.Fetched), nil
}
// One read per artist, however many ask at once, on its own context.
done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("reading the description panicked: %v", r)
}
}()
return nil, s.refresh(artist)
})
select {
case <-ctx.Done():
return Result{}, ctx.Err()
case r := <-done:
if errors.Is(r.Err, discovery.ErrNotFound) {
return Result{}, discovery.ErrNotFound // removed while it was being read
}
if errors.Is(r.Err, errSwitchedOff) {
return result(snap, discovery.StatusDisabled, snap.Fetched), nil
}
if r.Err != nil {
s.log.Warn("the description could not be read", zap.String("provider", Provider), zap.Error(r.Err))
return result(snap, discovery.StatusError, snap.Fetched), nil
}
}
snap, err = s.store.Description(ctx, artistID)
if err != nil {
return Result{}, err
}
return result(snap, discovery.StatusOK, false), nil
}
// errSwitchedOff is what a read returns when the providers were switched off while it ran.
var errSwitchedOff = errors.New("the provider was switched off")
// active says whether the chain may be followed: Wikipedia is on, and so is MusicBrainz,
// which gives the link to Wikidata. Without that second switch nothing would stop the
// artist's id going to a provider the admin never allowed.
func (s *Service) active(ctx context.Context) (bool, error) {
for _, name := range []string{Provider, providers.MusicBrainz} {
_, on, err := s.settings.Active(ctx, name)
if err != nil {
return false, err
}
if !on {
return false, nil
}
}
return true, nil
}
func (s *Service) fresh(snap Snapshot) bool {
ttl := PositiveTTL
if snap.None {
ttl = NegativeTTL
}
return s.now().Sub(snap.RetrievedAt) < ttl
}
func result(snap Snapshot, status string, stale bool) Result {
r := Result{Status: status, Stale: stale && snap.Fetched}
if snap.Fetched {
r.RetrievedAt = snap.RetrievedAt
if !snap.None {
d := snap.Description
r.Description = &d
}
}
return r
}
// refresh follows the chain and stores what it finds. A failure is returned and not
// stored: the next request tries again, and the language that failed is not replaced
// by the next one.
func (s *Service) refresh(a discovery.Artist) error {
ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
defer cancel()
item, err := s.source.WikidataID(ctx, a.MBID)
if err != nil && !errors.Is(err, ErrSourceNotFound) {
return err
}
if item == "" || err != nil {
return s.save(ctx, a.ID, "", nil)
}
resolved, pages, err := s.source.Pages(ctx, item, s.languages)
if errors.Is(err, ErrSourceNotFound) {
return s.save(ctx, a.ID, item, nil)
}
if err != nil {
return err
}
for _, lang := range s.languages {
title := pages[lang]
if title == "" {
continue
}
sum, err := s.source.Summary(ctx, lang, title)
if errors.Is(err, ErrSourceNotFound) {
continue
}
if err != nil {
return err
}
text := strings.TrimSpace(sum.Extract)
// An article about this very item (the one the pages belong to, which is the
// target when the item was merged), with some text, and an address to credit.
// Anything else (a page of disambiguation, a redirect to another subject, a page
// with no text) is not a description.
if sum.Type != "standard" || sum.WikibaseItem != resolved || text == "" || sum.URL == "" {
continue
}
return s.save(ctx, a.ID, resolved, &Description{
Text: cut(text), Short: sum.Description, Language: lang, Title: title, URL: sum.URL,
})
}
return s.save(ctx, a.ID, resolved, nil)
}
// save stores the outcome, unless the providers were switched off since the read began:
// an admin who took the data back and switched it off must not see it come back.
func (s *Service) save(ctx context.Context, id int64, item string, d *Description) error {
if on, err := s.active(ctx); err != nil {
return err
} else if !on {
return errSwitchedOff
}
return s.store.SaveDescription(ctx, id, s.now(), item, d)
}
// cut keeps at most MaxTextLen characters and marks a text that was shortened.
func cut(s string) string {
if utf8.RuneCountInString(s) <= MaxTextLen {
return s
}
return string([]rune(s)[:MaxTextLen-1]) + "…" // the mark counts: never more than MaxTextLen
}