// Package about tells what an artist is, with a short text from Wikipedia. // // The way to the text is an identifier chain, never a search by name: the artist's // MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the // Wikipedia page that Wikidata names in the user's languages. Only the id is // trusted: the page is used only if it says it is about that same item. // // Like the other provider data, it is asked for only when the admin switched the // provider on and someone asks, kept in the database (30 days; a day when there was // nothing to find), served stale when Wikipedia fails, and removed with the provider's // data. The text is CC BY-SA 4.0, so the page, its link and the licence are kept and // returned with it. package about import ( "context" "errors" "fmt" "strings" "time" "unicode/utf8" "go.uber.org/zap" "golang.org/x/sync/singleflight" "git.kmalloc.org/squid/sonda/internal/discovery" "git.kmalloc.org/squid/sonda/internal/providers" ) // Provider is the provider the descriptions come from. const Provider = providers.Wikipedia // What has to be shown with the text. const ( Attribution = "Text from Wikipedia (https://www.wikipedia.org), available under the CC BY-SA 4.0 licence." LicenseName = "CC BY-SA 4.0" LicenseURL = "https://creativecommons.org/licenses/by-sa/4.0/" ) // How long what was read is believed. A description changes slowly; "nothing found" // is remembered shortly, in case a link was added since. const ( PositiveTTL = 30 * 24 * time.Hour NegativeTTL = 24 * time.Hour ) // MaxTextLen is the longest text kept, in characters. A summary is a paragraph. const MaxTextLen = 4000 // fetchTimeout bounds a shared read, which does not depend on whoever started it // still waiting. It covers up to three requests, the first of them to MusicBrainz at // one a second. const fetchTimeout = 45 * time.Second // Errors a Source returns. var ( // ErrSourceNotFound means the provider does not know what was asked: an unknown // artist, item or page. It is an answer, not a failure. ErrSourceNotFound = errors.New("the provider does not know this") // ErrSourceUnavailable stands for any failure to get an answer. ErrSourceUnavailable = errors.New("the provider could not be reached") ) // Description is what Wikipedia says about an artist, with where it comes from. type Description struct { Text string // the first paragraph of the article, plain text Short string // one line, like "English rock band"; may be empty Language string // of the page: "it", "en" Title string // of the page URL string // of the page, on Wikipedia } // Snapshot is what is stored for one artist. Fetched is false when the provider was // never asked; None when it was and there was nothing to find. type Snapshot struct { Fetched bool None bool RetrievedAt time.Time Description Description } // Result is the answer to Description. RetrievedAt is zero when nothing is known; Stale // means the description is older than it should be because Wikipedia could not be asked. type Result struct { Status string // discovery.StatusOK, StatusDisabled, StatusError or StatusNoID Stale bool RetrievedAt time.Time Description *Description // nil when there is none } // Store keeps the descriptions. Artist returns discovery.ErrNotFound for an id the // catalog does not know, and so does SaveDescription for an artist removed meanwhile. type Store interface { Artist(ctx context.Context, id int64) (discovery.Artist, error) Description(ctx context.Context, id int64) (Snapshot, error) // SaveDescription replaces what is stored; a nil description records that there is // none. wikidataID is the item MusicBrainz linked, or "". SaveDescription(ctx context.Context, id int64, at time.Time, wikidataID string, d *Description) error } // Summary is what a Wikipedia page says about itself. type Summary struct { Type string Title string Lang string Extract string Description string URL string WikibaseItem string } // Source follows the chain of identifiers. Every method returns ErrSourceNotFound for // what the provider does not know, and any other error for a failure. type Source interface { // WikidataID returns the Wikidata item MusicBrainz links to the artist, or "". WikidataID(ctx context.Context, musicbrainzID string) (string, error) // Pages returns the title of the page of the item in each of the languages that has // one, and the item they belong to: the same, or the one it was merged into. Pages(ctx context.Context, item string, languages []string) (string, map[string]string, error) Summary(ctx context.Context, language, title string) (Summary, error) } // Service answers "what is this artist". type Service struct { store Store source Source settings discovery.Settings languages []string log *zap.Logger now func() time.Time flight singleflight.Group } // NewService builds a Service. languages are the Wikipedia languages to try, in order. func NewService(store Store, source Source, settings discovery.Settings, languages []string, log *zap.Logger) *Service { if log == nil { log = zap.NewNop() } return &Service{store: store, source: source, settings: settings, languages: languages, log: log, now: time.Now} } // Description returns the description of an artist. A provider that is off, cannot be // asked or fails is not an error: the answer says so and carries what was stored. func (s *Service) Description(ctx context.Context, artistID int64) (Result, error) { return s.describe(ctx, artistID, false) } // Ensure is Description for the job that goes through the library: with refresh it reads // again even a description that is still good. Whatever goes wrong is in the status of the // answer, which then still carries what was stored: a refresh never replaces a description // by a failure. func (s *Service) Ensure(ctx context.Context, artistID int64, refresh bool) (Result, error) { return s.describe(ctx, artistID, refresh) } func (s *Service) describe(ctx context.Context, artistID int64, force bool) (Result, error) { if artistID <= 0 { return Result{}, discovery.ErrNotFound } artist, err := s.store.Artist(ctx, artistID) if err != nil { return Result{}, err } snap, err := s.store.Description(ctx, artistID) if err != nil { return Result{}, err } fresh := !force && snap.Fetched && s.fresh(snap) active, err := s.active(ctx) if err != nil { return Result{}, err } switch { case !active: return result(snap, discovery.StatusDisabled, snap.Fetched && !fresh), nil case fresh: return result(snap, discovery.StatusOK, false), nil } if artist.MBID == "" { // Sonda never looks an id up by name return result(snap, discovery.StatusNoID, snap.Fetched), nil } // One read per artist, however many ask at once, on its own context. done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("reading the description panicked: %v", r) } }() return nil, s.refresh(artist) }) select { case <-ctx.Done(): return Result{}, ctx.Err() case r := <-done: if errors.Is(r.Err, discovery.ErrNotFound) { return Result{}, discovery.ErrNotFound // removed while it was being read } if errors.Is(r.Err, errSwitchedOff) { return result(snap, discovery.StatusDisabled, snap.Fetched), nil } if r.Err != nil { s.log.Warn("the description could not be read", zap.String("provider", Provider), zap.Error(r.Err)) return result(snap, discovery.StatusError, snap.Fetched), nil } } snap, err = s.store.Description(ctx, artistID) if err != nil { return Result{}, err } return result(snap, discovery.StatusOK, false), nil } // errSwitchedOff is what a read returns when the providers were switched off while it ran. var errSwitchedOff = errors.New("the provider was switched off") // active says whether the chain may be followed: Wikipedia is on, and so is MusicBrainz, // which gives the link to Wikidata. Without that second switch nothing would stop the // artist's id going to a provider the admin never allowed. func (s *Service) active(ctx context.Context) (bool, error) { for _, name := range []string{Provider, providers.MusicBrainz} { _, on, err := s.settings.Active(ctx, name) if err != nil { return false, err } if !on { return false, nil } } return true, nil } func (s *Service) fresh(snap Snapshot) bool { ttl := PositiveTTL if snap.None { ttl = NegativeTTL } return s.now().Sub(snap.RetrievedAt) < ttl } func result(snap Snapshot, status string, stale bool) Result { r := Result{Status: status, Stale: stale && snap.Fetched} if snap.Fetched { r.RetrievedAt = snap.RetrievedAt if !snap.None { d := snap.Description r.Description = &d } } return r } // refresh follows the chain and stores what it finds. A failure is returned and not // stored: the next request tries again, and the language that failed is not replaced // by the next one. func (s *Service) refresh(a discovery.Artist) error { ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) defer cancel() item, err := s.source.WikidataID(ctx, a.MBID) if err != nil && !errors.Is(err, ErrSourceNotFound) { return err } if item == "" || err != nil { return s.save(ctx, a.ID, "", nil) } resolved, pages, err := s.source.Pages(ctx, item, s.languages) if errors.Is(err, ErrSourceNotFound) { return s.save(ctx, a.ID, item, nil) } if err != nil { return err } for _, lang := range s.languages { title := pages[lang] if title == "" { continue } sum, err := s.source.Summary(ctx, lang, title) if errors.Is(err, ErrSourceNotFound) { continue } if err != nil { return err } text := strings.TrimSpace(sum.Extract) // An article about this very item (the one the pages belong to, which is the // target when the item was merged), with some text, and an address to credit. // Anything else (a page of disambiguation, a redirect to another subject, a page // with no text) is not a description. if sum.Type != "standard" || sum.WikibaseItem != resolved || text == "" || sum.URL == "" { continue } return s.save(ctx, a.ID, resolved, &Description{ Text: cut(text), Short: sum.Description, Language: lang, Title: title, URL: sum.URL, }) } return s.save(ctx, a.ID, resolved, nil) } // save stores the outcome, unless the providers were switched off since the read began: // an admin who took the data back and switched it off must not see it come back. func (s *Service) save(ctx context.Context, id int64, item string, d *Description) error { if on, err := s.active(ctx); err != nil { return err } else if !on { return errSwitchedOff } return s.store.SaveDescription(ctx, id, s.now(), item, d) } // cut keeps at most MaxTextLen characters and marks a text that was shortened. func cut(s string) string { if utf8.RuneCountInString(s) <= MaxTextLen { return s } return string([]rune(s)[:MaxTextLen-1]) + "…" // the mark counts: never more than MaxTextLen }