Add one engine, internal/jobs, that runs the jobs which go through the whole library: start, stop, progress, one at a time, give up after five failures in a row, end at once when a provider is off. - The covers job now runs on the engine; its API is unchanged. - POST/GET/DELETE /library/artist-images looks for the picture of every artist of the library that has a MusicBrainz id. - POST/GET/DELETE /library/descriptions does the same for the Wikipedia description. - ?refresh=true searches everything again and never replaces a stored result by a failure; artists without an id are never touched, nothing is looked up by name. - images.Service.EnsureArtistImage and about.Service.Ensure do the work for one artist; the lists come from ImageRepo and DiscoveryRepo. - jobs.Runner binds a job to the application context, replacing the two hand-written controls. - OpenAPI, contract test and README describe the new endpoints and say that they are bulk requests.
318 lines
11 KiB
Go
318 lines
11 KiB
Go
// Package about tells what an artist is, with a short text from Wikipedia.
|
|
//
|
|
// The way to the text is an identifier chain, never a search by name: the artist's
|
|
// MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the
|
|
// Wikipedia page that Wikidata names in the user's languages. Only the id is
|
|
// trusted: the page is used only if it says it is about that same item.
|
|
//
|
|
// Like the other provider data, it is asked for only when the admin switched the
|
|
// provider on and someone asks, kept in the database (30 days; a day when there was
|
|
// nothing to find), served stale when Wikipedia fails, and removed with the provider's
|
|
// data. The text is CC BY-SA 4.0, so the page, its link and the licence are kept and
|
|
// returned with it.
|
|
package about
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"go.uber.org/zap"
|
|
"golang.org/x/sync/singleflight"
|
|
|
|
"codeberg.org/squidd/sonda/internal/discovery"
|
|
"codeberg.org/squidd/sonda/internal/providers"
|
|
)
|
|
|
|
// Provider is the provider the descriptions come from.
|
|
const Provider = providers.Wikipedia
|
|
|
|
// What has to be shown with the text.
|
|
const (
|
|
Attribution = "Text from Wikipedia (https://www.wikipedia.org), available under the CC BY-SA 4.0 licence."
|
|
LicenseName = "CC BY-SA 4.0"
|
|
LicenseURL = "https://creativecommons.org/licenses/by-sa/4.0/"
|
|
)
|
|
|
|
// How long what was read is believed. A description changes slowly; "nothing found"
|
|
// is remembered shortly, in case a link was added since.
|
|
const (
|
|
PositiveTTL = 30 * 24 * time.Hour
|
|
NegativeTTL = 24 * time.Hour
|
|
)
|
|
|
|
// MaxTextLen is the longest text kept, in characters. A summary is a paragraph.
|
|
const MaxTextLen = 4000
|
|
|
|
// fetchTimeout bounds a shared read, which does not depend on whoever started it
|
|
// still waiting. It covers up to three requests, the first of them to MusicBrainz at
|
|
// one a second.
|
|
const fetchTimeout = 45 * time.Second
|
|
|
|
// Errors a Source returns.
|
|
var (
|
|
// ErrSourceNotFound means the provider does not know what was asked: an unknown
|
|
// artist, item or page. It is an answer, not a failure.
|
|
ErrSourceNotFound = errors.New("the provider does not know this")
|
|
// ErrSourceUnavailable stands for any failure to get an answer.
|
|
ErrSourceUnavailable = errors.New("the provider could not be reached")
|
|
)
|
|
|
|
// Description is what Wikipedia says about an artist, with where it comes from.
|
|
type Description struct {
|
|
Text string // the first paragraph of the article, plain text
|
|
Short string // one line, like "English rock band"; may be empty
|
|
Language string // of the page: "it", "en"
|
|
Title string // of the page
|
|
URL string // of the page, on Wikipedia
|
|
}
|
|
|
|
// Snapshot is what is stored for one artist. Fetched is false when the provider was
|
|
// never asked; None when it was and there was nothing to find.
|
|
type Snapshot struct {
|
|
Fetched bool
|
|
None bool
|
|
RetrievedAt time.Time
|
|
Description Description
|
|
}
|
|
|
|
// Result is the answer to Description. RetrievedAt is zero when nothing is known; Stale
|
|
// means the description is older than it should be because Wikipedia could not be asked.
|
|
type Result struct {
|
|
Status string // discovery.StatusOK, StatusDisabled, StatusError or StatusNoID
|
|
Stale bool
|
|
RetrievedAt time.Time
|
|
Description *Description // nil when there is none
|
|
}
|
|
|
|
// Store keeps the descriptions. Artist returns discovery.ErrNotFound for an id the
|
|
// catalog does not know, and so does SaveDescription for an artist removed meanwhile.
|
|
type Store interface {
|
|
Artist(ctx context.Context, id int64) (discovery.Artist, error)
|
|
Description(ctx context.Context, id int64) (Snapshot, error)
|
|
// SaveDescription replaces what is stored; a nil description records that there is
|
|
// none. wikidataID is the item MusicBrainz linked, or "".
|
|
SaveDescription(ctx context.Context, id int64, at time.Time, wikidataID string, d *Description) error
|
|
}
|
|
|
|
// Summary is what a Wikipedia page says about itself.
|
|
type Summary struct {
|
|
Type string
|
|
Title string
|
|
Lang string
|
|
Extract string
|
|
Description string
|
|
URL string
|
|
WikibaseItem string
|
|
}
|
|
|
|
// Source follows the chain of identifiers. Every method returns ErrSourceNotFound for
|
|
// what the provider does not know, and any other error for a failure.
|
|
type Source interface {
|
|
// WikidataID returns the Wikidata item MusicBrainz links to the artist, or "".
|
|
WikidataID(ctx context.Context, musicbrainzID string) (string, error)
|
|
// Pages returns the title of the page of the item in each of the languages that has
|
|
// one, and the item they belong to: the same, or the one it was merged into.
|
|
Pages(ctx context.Context, item string, languages []string) (string, map[string]string, error)
|
|
Summary(ctx context.Context, language, title string) (Summary, error)
|
|
}
|
|
|
|
// Service answers "what is this artist".
|
|
type Service struct {
|
|
store Store
|
|
source Source
|
|
settings discovery.Settings
|
|
languages []string
|
|
log *zap.Logger
|
|
now func() time.Time
|
|
flight singleflight.Group
|
|
}
|
|
|
|
// NewService builds a Service. languages are the Wikipedia languages to try, in order.
|
|
func NewService(store Store, source Source, settings discovery.Settings, languages []string, log *zap.Logger) *Service {
|
|
if log == nil {
|
|
log = zap.NewNop()
|
|
}
|
|
return &Service{store: store, source: source, settings: settings, languages: languages, log: log, now: time.Now}
|
|
}
|
|
|
|
// Description returns the description of an artist. A provider that is off, cannot be
|
|
// asked or fails is not an error: the answer says so and carries what was stored.
|
|
func (s *Service) Description(ctx context.Context, artistID int64) (Result, error) {
|
|
return s.describe(ctx, artistID, false)
|
|
}
|
|
|
|
// Ensure is Description for the job that goes through the library: with refresh it reads
|
|
// again even a description that is still good. Whatever goes wrong is in the status of the
|
|
// answer, which then still carries what was stored: a refresh never replaces a description
|
|
// by a failure.
|
|
func (s *Service) Ensure(ctx context.Context, artistID int64, refresh bool) (Result, error) {
|
|
return s.describe(ctx, artistID, refresh)
|
|
}
|
|
|
|
func (s *Service) describe(ctx context.Context, artistID int64, force bool) (Result, error) {
|
|
if artistID <= 0 {
|
|
return Result{}, discovery.ErrNotFound
|
|
}
|
|
artist, err := s.store.Artist(ctx, artistID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
snap, err := s.store.Description(ctx, artistID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
fresh := !force && snap.Fetched && s.fresh(snap)
|
|
active, err := s.active(ctx)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
switch {
|
|
case !active:
|
|
return result(snap, discovery.StatusDisabled, snap.Fetched && !fresh), nil
|
|
case fresh:
|
|
return result(snap, discovery.StatusOK, false), nil
|
|
}
|
|
if artist.MBID == "" { // Sonda never looks an id up by name
|
|
return result(snap, discovery.StatusNoID, snap.Fetched), nil
|
|
}
|
|
|
|
// One read per artist, however many ask at once, on its own context.
|
|
done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
err = fmt.Errorf("reading the description panicked: %v", r)
|
|
}
|
|
}()
|
|
return nil, s.refresh(artist)
|
|
})
|
|
select {
|
|
case <-ctx.Done():
|
|
return Result{}, ctx.Err()
|
|
case r := <-done:
|
|
if errors.Is(r.Err, discovery.ErrNotFound) {
|
|
return Result{}, discovery.ErrNotFound // removed while it was being read
|
|
}
|
|
if errors.Is(r.Err, errSwitchedOff) {
|
|
return result(snap, discovery.StatusDisabled, snap.Fetched), nil
|
|
}
|
|
if r.Err != nil {
|
|
s.log.Warn("the description could not be read", zap.String("provider", Provider), zap.Error(r.Err))
|
|
return result(snap, discovery.StatusError, snap.Fetched), nil
|
|
}
|
|
}
|
|
snap, err = s.store.Description(ctx, artistID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
return result(snap, discovery.StatusOK, false), nil
|
|
}
|
|
|
|
// errSwitchedOff is what a read returns when the providers were switched off while it ran.
|
|
var errSwitchedOff = errors.New("the provider was switched off")
|
|
|
|
// active says whether the chain may be followed: Wikipedia is on, and so is MusicBrainz,
|
|
// which gives the link to Wikidata. Without that second switch nothing would stop the
|
|
// artist's id going to a provider the admin never allowed.
|
|
func (s *Service) active(ctx context.Context) (bool, error) {
|
|
for _, name := range []string{Provider, providers.MusicBrainz} {
|
|
_, on, err := s.settings.Active(ctx, name)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !on {
|
|
return false, nil
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (s *Service) fresh(snap Snapshot) bool {
|
|
ttl := PositiveTTL
|
|
if snap.None {
|
|
ttl = NegativeTTL
|
|
}
|
|
return s.now().Sub(snap.RetrievedAt) < ttl
|
|
}
|
|
|
|
func result(snap Snapshot, status string, stale bool) Result {
|
|
r := Result{Status: status, Stale: stale && snap.Fetched}
|
|
if snap.Fetched {
|
|
r.RetrievedAt = snap.RetrievedAt
|
|
if !snap.None {
|
|
d := snap.Description
|
|
r.Description = &d
|
|
}
|
|
}
|
|
return r
|
|
}
|
|
|
|
// refresh follows the chain and stores what it finds. A failure is returned and not
|
|
// stored: the next request tries again, and the language that failed is not replaced
|
|
// by the next one.
|
|
func (s *Service) refresh(a discovery.Artist) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
|
|
defer cancel()
|
|
|
|
item, err := s.source.WikidataID(ctx, a.MBID)
|
|
if err != nil && !errors.Is(err, ErrSourceNotFound) {
|
|
return err
|
|
}
|
|
if item == "" || err != nil {
|
|
return s.save(ctx, a.ID, "", nil)
|
|
}
|
|
resolved, pages, err := s.source.Pages(ctx, item, s.languages)
|
|
if errors.Is(err, ErrSourceNotFound) {
|
|
return s.save(ctx, a.ID, item, nil)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, lang := range s.languages {
|
|
title := pages[lang]
|
|
if title == "" {
|
|
continue
|
|
}
|
|
sum, err := s.source.Summary(ctx, lang, title)
|
|
if errors.Is(err, ErrSourceNotFound) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
text := strings.TrimSpace(sum.Extract)
|
|
// An article about this very item (the one the pages belong to, which is the
|
|
// target when the item was merged), with some text, and an address to credit.
|
|
// Anything else (a page of disambiguation, a redirect to another subject, a page
|
|
// with no text) is not a description.
|
|
if sum.Type != "standard" || sum.WikibaseItem != resolved || text == "" || sum.URL == "" {
|
|
continue
|
|
}
|
|
return s.save(ctx, a.ID, resolved, &Description{
|
|
Text: cut(text), Short: sum.Description, Language: lang, Title: title, URL: sum.URL,
|
|
})
|
|
}
|
|
return s.save(ctx, a.ID, resolved, nil)
|
|
}
|
|
|
|
// save stores the outcome, unless the providers were switched off since the read began:
|
|
// an admin who took the data back and switched it off must not see it come back.
|
|
func (s *Service) save(ctx context.Context, id int64, item string, d *Description) error {
|
|
if on, err := s.active(ctx); err != nil {
|
|
return err
|
|
} else if !on {
|
|
return errSwitchedOff
|
|
}
|
|
return s.store.SaveDescription(ctx, id, s.now(), item, d)
|
|
}
|
|
|
|
// cut keeps at most MaxTextLen characters and marks a text that was shortened.
|
|
func cut(s string) string {
|
|
if utf8.RuneCountInString(s) <= MaxTextLen {
|
|
return s
|
|
}
|
|
return string([]rune(s)[:MaxTextLen-1]) + "…" // the mark counts: never more than MaxTextLen
|
|
}
|